From 23c9913ef7f8ed8835165fc6c65617e3090f1630 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 28 Apr 2026 16:26:10 -0700 Subject: [PATCH 001/350] Add option to specify an arena and persist the resolved types from type checking. PiperOrigin-RevId: 907236091 --- checker/BUILD | 5 ++++ checker/internal/type_checker_impl.cc | 34 +++++++++++++++++++------ checker/internal/type_checker_impl.h | 4 +-- checker/type_checker.cc | 36 +++++++++++++++++++++++++++ checker/type_checker.h | 15 ++++++++--- checker/validation_result.h | 20 ++++++++++++++- compiler/BUILD | 2 ++ compiler/compiler.h | 12 +++++++-- compiler/compiler_factory.cc | 7 +++--- compiler/compiler_factory_test.cc | 23 +++++++++++++++++ 10 files changed, 139 insertions(+), 19 deletions(-) create mode 100644 checker/type_checker.cc diff --git a/checker/BUILD b/checker/BUILD index f1e0cef3c..27a1eb84e 100644 --- a/checker/BUILD +++ b/checker/BUILD @@ -50,7 +50,9 @@ cc_library( ":type_check_issue", "//common:ast", "//common:source", + "//common:type", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -74,11 +76,14 @@ cc_test( cc_library( name = "type_checker", + srcs = ["type_checker.cc"], hdrs = ["type_checker.h"], deps = [ ":validation_result", "//common:ast", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status:statusor", + "@com_google_protobuf//:protobuf", ], ) diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 8f67efbde..05601fdbb 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1176,11 +1177,13 @@ class ResolveRewriter : public AstRewriterBase { explicit ResolveRewriter(const ResolveVisitor& visitor, const TypeInferenceContext& inference_context, const CheckerOptions& options, - Ast::ReferenceMap& references, Ast::TypeMap& types) + Ast::ReferenceMap& references, Ast::TypeMap& types, + ValidationResult::TypeMap& resolved_types) : visitor_(visitor), inference_context_(inference_context), reference_map_(references), type_map_(types), + resolved_types_(resolved_types), options_(options) {} bool PostVisitRewrite(Expr& expr) override { bool rewritten = false; @@ -1235,6 +1238,7 @@ class ResolveRewriter : public AstRewriterBase { return rewritten; } type_map_[expr.id()] = *std::move(flattened_type); + resolved_types_[expr.id()] = iter->second; rewritten = true; } @@ -1249,23 +1253,28 @@ class ResolveRewriter : public AstRewriterBase { const TypeInferenceContext& inference_context_; Ast::ReferenceMap& reference_map_; Ast::TypeMap& type_map_; + ValidationResult::TypeMap& resolved_types_; const CheckerOptions& options_; }; } // namespace -absl::StatusOr TypeCheckerImpl::Check( - std::unique_ptr ast) const { - google::protobuf::Arena type_arena; +absl::StatusOr TypeCheckerImpl::CheckImpl( + std::unique_ptr ast, google::protobuf::Arena* arena) const { + std::optional type_arena; + if (arena == nullptr) { + type_arena.emplace(); + arena = &(*type_arena); + } std::vector issues; CEL_ASSIGN_OR_RETURN(auto generator, NamespaceGenerator::Create(env_.container())); TypeInferenceContext type_inference_context( - &type_arena, options_.enable_legacy_null_assignment); + arena, options_.enable_legacy_null_assignment); ResolveVisitor visitor(std::move(generator), env_, *ast, - type_inference_context, issues, &type_arena); + type_inference_context, issues, arena); TraversalOptions opts; opts.use_comprehension_callbacks = true; @@ -1310,9 +1319,10 @@ absl::StatusOr TypeCheckerImpl::Check( // Apply updates as needed. // Happens in a second pass to simplify validating that pointers haven't // been invalidated by other updates. + ValidationResult::TypeMap resolved_types; ResolveRewriter rewriter(visitor, type_inference_context, options_, ast->mutable_reference_map(), - ast->mutable_type_map()); + ast->mutable_type_map(), resolved_types); AstRewrite(ast->mutable_root_expr(), rewriter); CEL_RETURN_IF_ERROR(rewriter.status()); @@ -1325,7 +1335,15 @@ absl::StatusOr TypeCheckerImpl::Check( {cel::ExtensionSpec::Component::kRuntime})); } - return ValidationResult(std::move(ast), std::move(issues)); + auto result = ValidationResult(std::move(ast), std::move(issues)); + if (!type_arena.has_value()) { + // cel::Type values will expire after this function returns when the local + // arena is destructed. Only set the resolved type map if we're using the + // caller's arena. + result.SetResolvedTypeMap(std::move(resolved_types)); + } + + return result; } std::unique_ptr TypeCheckerImpl::ToBuilder() const { diff --git a/checker/internal/type_checker_impl.h b/checker/internal/type_checker_impl.h index 71683276d..9ee9a50d0 100644 --- a/checker/internal/type_checker_impl.h +++ b/checker/internal/type_checker_impl.h @@ -42,8 +42,8 @@ class TypeCheckerImpl : public TypeChecker { TypeCheckerImpl(TypeCheckerImpl&&) = delete; TypeCheckerImpl& operator=(TypeCheckerImpl&&) = delete; - absl::StatusOr Check( - std::unique_ptr ast) const override; + absl::StatusOr CheckImpl( + std::unique_ptr ast, google::protobuf::Arena* arena) const override; std::unique_ptr ToBuilder() const override; diff --git a/checker/type_checker.cc b/checker/type_checker.cc new file mode 100644 index 000000000..6d59e144d --- /dev/null +++ b/checker/type_checker.cc @@ -0,0 +1,36 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/type_checker.h" + +namespace cel { +absl::StatusOr TypeChecker::Check( + std::unique_ptr ast) const { + return CheckImpl(std::move(ast), nullptr); +} + +absl::StatusOr TypeChecker::Check( + std::unique_ptr ast, google::protobuf::Arena* arena) const { + return CheckImpl(std::move(ast), arena); +} + +absl::StatusOr TypeChecker::Check(const Ast& ast) const { + return CheckImpl(std::make_unique(ast), nullptr); +} + +absl::StatusOr TypeChecker::Check( + const Ast& ast, google::protobuf::Arena* arena) const { + return CheckImpl(std::make_unique(ast), arena); +} +} // namespace cel diff --git a/checker/type_checker.h b/checker/type_checker.h index e47b7dca6..edb6cc91f 100644 --- a/checker/type_checker.h +++ b/checker/type_checker.h @@ -16,10 +16,13 @@ #define THIRD_PARTY_CEL_CPP_CHECKER_TYPE_CHECKER_H_ #include +#include +#include "absl/base/nullability.h" #include "absl/status/statusor.h" #include "checker/validation_result.h" #include "common/ast.h" +#include "google/protobuf/arena.h" namespace cel { @@ -42,13 +45,19 @@ class TypeChecker { // A non-ok status is returned if type checking can't reasonably complete // (e.g. if an internal precondition is violated or an extension returns an // error). - virtual absl::StatusOr Check( - std::unique_ptr ast) const = 0; + absl::StatusOr Check(std::unique_ptr ast) const; + absl::StatusOr Check(std::unique_ptr ast, + google::protobuf::Arena* arena) const; + absl::StatusOr Check(const Ast& ast) const; + absl::StatusOr Check(const Ast& ast, + google::protobuf::Arena* arena) const; // Returns a builder initialized with the configuration of this type checker. virtual std::unique_ptr ToBuilder() const = 0; - // TODO(uncreated-issue/73): add overload for cref AST. + private: + virtual absl::StatusOr CheckImpl( + std::unique_ptr ast, google::protobuf::Arena* absl_nullable arena) const = 0; }; } // namespace cel diff --git a/checker/validation_result.h b/checker/validation_result.h index 8c84a84da..f424e7f6f 100644 --- a/checker/validation_result.h +++ b/checker/validation_result.h @@ -15,26 +15,31 @@ #ifndef THIRD_PARTY_CEL_CPP_CHECKER_VALIDATION_RESULT_H_ #define THIRD_PARTY_CEL_CPP_CHECKER_VALIDATION_RESULT_H_ +#include #include #include #include #include #include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/types/span.h" #include "checker/type_check_issue.h" #include "common/ast.h" #include "common/source.h" +#include "common/type.h" namespace cel { -// ValidationResult holds the result of TypeChecking. +// ValidationResult holds the result of type checking. // // Error states are captured as type check issues where possible. class ValidationResult { public: + using TypeMap = absl::flat_hash_map; + ValidationResult(std::unique_ptr ast, std::vector issues) : ast_(std::move(ast)), issues_(std::move(issues)) {} @@ -71,6 +76,18 @@ class ValidationResult { return std::move(source_); } + // Returns the resolved type map for the AST. + // + // Only populated if the AST was checked with an explicit arena. + // + // The type entries may have storage in the arena or reference type + // information from the type checker that produced the AST. This means the map + // is only valid as long as both the type checker and the arena are valid. + const TypeMap& GetResolvedTypeMap() const { return resolved_type_map_; } + void SetResolvedTypeMap(TypeMap resolved_type_map) { + resolved_type_map_ = std::move(resolved_type_map); + } + // Returns a string representation of the issues in the result suitable for // display. // @@ -89,6 +106,7 @@ class ValidationResult { private: absl_nullable std::unique_ptr ast_; + TypeMap resolved_type_map_; std::vector issues_; absl_nullable std::unique_ptr source_; }; diff --git a/compiler/BUILD b/compiler/BUILD index 170f1068b..50bc1c9fa 100644 --- a/compiler/BUILD +++ b/compiler/BUILD @@ -28,9 +28,11 @@ cc_library( "//parser:options", "//parser:parser_interface", "//validator", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", + "@com_google_protobuf//:protobuf", ], ) diff --git a/compiler/compiler.h b/compiler/compiler.h index 48fa4e0b1..6d07e72c2 100644 --- a/compiler/compiler.h +++ b/compiler/compiler.h @@ -19,6 +19,7 @@ #include #include +#include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -29,6 +30,7 @@ #include "parser/options.h" #include "parser/parser_interface.h" #include "validator/validator.h" +#include "google/protobuf/arena.h" namespace cel { @@ -126,10 +128,16 @@ class Compiler { virtual ~Compiler() = default; virtual absl::StatusOr Compile( - absl::string_view source, absl::string_view description) const = 0; + absl::string_view source, absl::string_view description, + google::protobuf::Arena* absl_nullable arena) const = 0; absl::StatusOr Compile(absl::string_view source) const { - return Compile(source, ""); + return Compile(source, "", nullptr); + } + + absl::StatusOr Compile( + absl::string_view source, absl::string_view description) const { + return Compile(source, description, nullptr); } // Accessor for the underlying type checker. diff --git a/compiler/compiler_factory.cc b/compiler/compiler_factory.cc index 3e9871706..14586825e 100644 --- a/compiler/compiler_factory.cc +++ b/compiler/compiler_factory.cc @@ -33,6 +33,7 @@ #include "parser/parser.h" #include "parser/parser_interface.h" #include "validator/validator.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" namespace cel { @@ -50,13 +51,13 @@ class CompilerImpl : public Compiler { validator_(std::move(validator)) {} absl::StatusOr Compile( - absl::string_view expression, - absl::string_view description) const override { + absl::string_view expression, absl::string_view description, + google::protobuf::Arena* arena) const override { CEL_ASSIGN_OR_RETURN(auto source, cel::NewSource(expression, std::string(description))); CEL_ASSIGN_OR_RETURN(auto ast, parser_->Parse(*source)); CEL_ASSIGN_OR_RETURN(ValidationResult result, - type_checker_->Check(std::move(ast))); + type_checker_->Check(std::move(ast), arena)); result.SetSource(std::move(source)); if (!validator_.validations().empty()) { diff --git a/compiler/compiler_factory_test.cc b/compiler/compiler_factory_test.cc index d217e4cc7..214c23765 100644 --- a/compiler/compiler_factory_test.cc +++ b/compiler/compiler_factory_test.cc @@ -37,6 +37,7 @@ #include "parser/parser_interface.h" #include "testutil/baseline_tests.h" #include "validator/timestamp_literal_validator.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" namespace cel { @@ -390,5 +391,27 @@ TEST(CompilerFactoryTest, ToBuilderWorks) { EXPECT_TRUE(result.IsValid()); } +TEST(CompilerFactoryTest, SpecifyArenaKeepsResolvedTypes) { + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool())); + + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->AddLibrary(OptionalCompilerLibrary()), IsOk()); + + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("[[1, 2, 3]][?0]", "", &arena)); + ASSERT_OK_AND_ASSIGN(auto ast, result.ReleaseAst()); + auto it = result.GetResolvedTypeMap().find(ast->root_expr().id()); + ASSERT_TRUE(it != result.GetResolvedTypeMap().end()); + EXPECT_TRUE( + it->second.IsOptional() && + it->second.GetOptional().GetParameter().IsList() && + it->second.GetOptional().GetParameter().GetList().GetElement().IsInt()); +} + } // namespace } // namespace cel From 9e73d93f77a159a149edd9a465d092cff03d702a Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 30 Apr 2026 13:04:30 -0700 Subject: [PATCH 002/350] Deprecate option to switch to old accumulator var name in parser. cel::ParserOption::enable_hidden_accumulator_var now has no effect and will be removed in a later change. The standard / extension macros should always use `@result` now. PiperOrigin-RevId: 908333116 --- common/expr.h | 4 +- common/expr_factory.h | 1 - extensions/comprehensions_v2_macros.cc | 72 +++---- parser/macro.cc | 44 ++-- parser/macro_expr_factory.h | 3 +- parser/macro_expr_factory_test.cc | 2 +- parser/options.h | 4 +- parser/parser.cc | 19 +- parser/parser_test.cc | 270 ------------------------- 9 files changed, 72 insertions(+), 347 deletions(-) diff --git a/common/expr.h b/common/expr.h index 9c6f508c6..7305c2c9f 100644 --- a/common/expr.h +++ b/common/expr.h @@ -45,7 +45,9 @@ class MapExprEntry; class MapExpr; class ComprehensionExpr; -inline constexpr absl::string_view kAccumulatorVariableName = "__result__"; +inline constexpr absl::string_view kAccumulatorVariableName = "@result"; +inline constexpr absl::string_view kDeprecatedAccumulatorVariableName = + "__result__"; bool operator==(const Expr& lhs, const Expr& rhs); diff --git a/common/expr_factory.h b/common/expr_factory.h index c8a9b831f..b9769b457 100644 --- a/common/expr_factory.h +++ b/common/expr_factory.h @@ -357,7 +357,6 @@ class ExprFactory { friend class ParserMacroExprFactory; ExprFactory() : accu_var_(kAccumulatorVariableName) {} - explicit ExprFactory(absl::string_view accu_var) : accu_var_(accu_var) {} std::string accu_var_; }; diff --git a/extensions/comprehensions_v2_macros.cc b/extensions/comprehensions_v2_macros.cc index a8de3a103..134fb80ff 100644 --- a/extensions/comprehensions_v2_macros.cc +++ b/extensions/comprehensions_v2_macros.cc @@ -56,15 +56,15 @@ absl::optional ExpandAllMacro2(MacroExprFactory& factory, Expr& target, args[0], "all() second variable must be different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("all() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("all() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewBoolConst(true); auto condition = @@ -102,15 +102,15 @@ absl::optional ExpandExistsMacro2(MacroExprFactory& factory, Expr& target, args[0], "exists() second variable must be different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("exists() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("exists() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewBoolConst(false); auto condition = factory.NewCall( @@ -153,15 +153,15 @@ absl::optional ExpandExistsOneMacro2(MacroExprFactory& factory, "existsOne() second variable must be different " "from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("existsOne() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("existsOne() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewIntConst(0); auto condition = factory.NewBoolConst(true); @@ -205,15 +205,15 @@ absl::optional ExpandTransformList3Macro(MacroExprFactory& factory, "transformList() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformList() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformList() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -254,15 +254,15 @@ absl::optional ExpandTransformList4Macro(MacroExprFactory& factory, "transformList() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformList() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformList() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -305,15 +305,15 @@ absl::optional ExpandTransformMap3Macro(MacroExprFactory& factory, "transformMap() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformMap() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformMap() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -353,15 +353,15 @@ absl::optional ExpandTransformMap4Macro(MacroExprFactory& factory, "transformMap() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformMap() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformMap() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -403,17 +403,17 @@ absl::optional ExpandTransformMapEntry3Macro(MacroExprFactory& factory, "transformMapEntry() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformMapEntry() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformMapEntry() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); @@ -453,17 +453,17 @@ absl::optional ExpandTransformMapEntry4Macro(MacroExprFactory& factory, "transformMapEntry() second variable must be " "different from the first variable"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[0], absl::StrCat("transformMapEntry() first variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } - if (args[1].ident_expr().name() == kAccumulatorVariableName) { + if (args[1].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("transformMapEntry() second variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } std::string iter_var = args[0].ident_expr().name(); std::string iter_var2 = args[1].ident_expr().name(); diff --git a/parser/macro.cc b/parser/macro.cc index eaa1ebd1a..8f8c9e596 100644 --- a/parser/macro.cc +++ b/parser/macro.cc @@ -91,10 +91,10 @@ absl::optional ExpandAllMacro(MacroExprFactory& factory, Expr& target, return factory.ReportErrorAt( args[0], "all() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { - return factory.ReportErrorAt(args[1], - absl::StrCat("all() variable name cannot be ", - kAccumulatorVariableName)); + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { + return factory.ReportErrorAt( + args[1], absl::StrCat("all() variable name cannot be ", + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewBoolConst(true); auto condition = @@ -123,10 +123,10 @@ absl::optional ExpandExistsMacro(MacroExprFactory& factory, Expr& target, return factory.ReportErrorAt( args[0], "exists() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("exists() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewBoolConst(false); auto condition = factory.NewCall( @@ -157,10 +157,10 @@ absl::optional ExpandExistsOneMacro(MacroExprFactory& factory, return factory.ReportErrorAt( args[0], "exists_one() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("exists_one() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewIntConst(0); auto condition = factory.NewBoolConst(true); @@ -196,10 +196,10 @@ absl::optional ExpandMap2Macro(MacroExprFactory& factory, Expr& target, return factory.ReportErrorAt( args[0], "map() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { - return factory.ReportErrorAt(args[1], - absl::StrCat("map() variable name cannot be ", - kAccumulatorVariableName)); + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { + return factory.ReportErrorAt( + args[1], absl::StrCat("map() variable name cannot be ", + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewList(); auto condition = factory.NewBoolConst(true); @@ -229,10 +229,10 @@ absl::optional ExpandMap3Macro(MacroExprFactory& factory, Expr& target, return factory.ReportErrorAt( args[0], "map() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { - return factory.ReportErrorAt(args[1], - absl::StrCat("map() variable name cannot be ", - kAccumulatorVariableName)); + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { + return factory.ReportErrorAt( + args[1], absl::StrCat("map() variable name cannot be ", + kDeprecatedAccumulatorVariableName)); } auto init = factory.NewList(); auto condition = factory.NewBoolConst(true); @@ -264,10 +264,10 @@ absl::optional ExpandFilterMacro(MacroExprFactory& factory, Expr& target, return factory.ReportErrorAt( args[0], "filter() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("filter() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto name = args[0].ident_expr().name(); @@ -302,10 +302,10 @@ absl::optional ExpandOptMapMacro(MacroExprFactory& factory, Expr& target, return factory.ReportErrorAt( args[0], "optMap() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("optMap() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto var_name = args[0].ident_expr().name(); @@ -341,10 +341,10 @@ absl::optional ExpandOptFlatMapMacro(MacroExprFactory& factory, return factory.ReportErrorAt( args[0], "optFlatMap() variable name must be a simple identifier"); } - if (args[0].ident_expr().name() == kAccumulatorVariableName) { + if (args[0].ident_expr().name() == kDeprecatedAccumulatorVariableName) { return factory.ReportErrorAt( args[1], absl::StrCat("optFlatMap() variable name cannot be ", - kAccumulatorVariableName)); + kDeprecatedAccumulatorVariableName)); } auto var_name = args[0].ident_expr().name(); diff --git a/parser/macro_expr_factory.h b/parser/macro_expr_factory.h index ffba5e2f2..c66aa4fe0 100644 --- a/parser/macro_expr_factory.h +++ b/parser/macro_expr_factory.h @@ -319,8 +319,7 @@ class MacroExprFactory : protected ExprFactory { friend class ParserMacroExprFactory; friend class TestMacroExprFactory; - explicit MacroExprFactory(absl::string_view accu_var) - : ExprFactory(accu_var) {} + explicit MacroExprFactory() = default; }; } // namespace cel diff --git a/parser/macro_expr_factory_test.cc b/parser/macro_expr_factory_test.cc index 04705eec6..489538be1 100644 --- a/parser/macro_expr_factory_test.cc +++ b/parser/macro_expr_factory_test.cc @@ -27,7 +27,7 @@ namespace cel { class TestMacroExprFactory final : public MacroExprFactory { public: - TestMacroExprFactory() : MacroExprFactory(kAccumulatorVariableName) {} + TestMacroExprFactory() = default; ExprId id() const { return id_; } diff --git a/parser/options.h b/parser/options.h index a41d16104..916a941f0 100644 --- a/parser/options.h +++ b/parser/options.h @@ -51,7 +51,9 @@ struct ParserOptions final { // Disable standard macros (has, all, exists, exists_one, filter, map). bool disable_standard_macros = false; - // Enable hidden accumulator variable '@result' for builtin comprehensions. + // Deprecated: The builtin and extension macros now always use the new + // accumulator variable name. + // This option has no effect. bool enable_hidden_accumulator_var = true; // Enables support for identifier quoting syntax: diff --git a/parser/parser.cc b/parser/parser.cc index d9f74e712..f4ee3a1c5 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -163,9 +163,8 @@ SourceRange SourceRangeFromParserRuleContext( class ParserMacroExprFactory final : public MacroExprFactory { public: - explicit ParserMacroExprFactory(const cel::Source& source, - absl::string_view accu_var) - : MacroExprFactory(accu_var), source_(source) {} + explicit ParserMacroExprFactory(const cel::Source& source) + : source_(source) {} void BeginMacro(SourceRange macro_position) { macro_position_ = macro_position; @@ -607,13 +606,12 @@ class ParserVisitor final : public CelBaseVisitor, public antlr4::BaseErrorListener { public: ParserVisitor(const cel::Source& source, int max_recursion_depth, - absl::string_view accu_var, const cel::MacroRegistry& macro_registry, bool add_macro_calls = false, bool enable_optional_syntax = false, bool enable_quoted_identifiers = false) : source_(source), - factory_(source_, accu_var), + factory_(source_), macro_registry_(macro_registry), recursion_depth_(0), max_recursion_depth_(max_recursion_depth), @@ -1654,14 +1652,9 @@ absl::StatusOr ParseImpl(const cel::Source& source, CommonTokenStream tokens(&lexer); CelParser parser(&tokens); ExprRecursionListener listener(options.max_recursion_depth); - absl::string_view accu_var = cel::kAccumulatorVariableName; - if (options.enable_hidden_accumulator_var) { - accu_var = cel::kHiddenAccumulatorVariableName; - } - ParserVisitor visitor(source, options.max_recursion_depth, accu_var, - registry, options.add_macro_calls, - options.enable_optional_syntax, - options.enable_quoted_identifiers); + ParserVisitor visitor( + source, options.max_recursion_depth, registry, options.add_macro_calls, + options.enable_optional_syntax, options.enable_quoted_identifiers); lexer.removeErrorListeners(); parser.removeErrorListeners(); diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 3659fd8fd..a1a65481d 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1473,7 +1473,6 @@ class ExpressionTest : public testing::TestWithParam {}; TEST_P(ExpressionTest, Parse) { const TestInfo& test_info = GetParam(); ParserOptions options; - options.enable_hidden_accumulator_var = true; if (!test_info.M.empty()) { options.add_macro_calls = true; } @@ -1628,271 +1627,6 @@ TEST(ExpressionTest, RecursionDepthIgnoresParentheses) { EXPECT_THAT(result, IsOk()); } -const std::vector& UpdatedAccuVarTestCases() { - static const std::vector* kInstance = new std::vector{ - {"[].exists(x, x > 0)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " false^#7:bool#,\n" - " // LoopCondition\n" - " @not_strictly_false(\n" - " !_(\n" - " __result__^#8:Expr.Ident#\n" - " )^#9:Expr.Call#\n" - " )^#10:Expr.Call#,\n" - " // LoopStep\n" - " _||_(\n" - " __result__^#11:Expr.Ident#,\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#\n" - " )^#12:Expr.Call#,\n" - " // Result\n" - " __result__^#13:Expr.Ident#)^#14:Expr.Comprehension#"}, - {"[].exists_one(x, x > 0)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " 0^#7:int64#,\n" - " // LoopCondition\n" - " true^#8:bool#,\n" - " // LoopStep\n" - " _?_:_(\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#,\n" - " _+_(\n" - " __result__^#9:Expr.Ident#,\n" - " 1^#10:int64#\n" - " )^#11:Expr.Call#,\n" - " __result__^#12:Expr.Ident#\n" - " )^#13:Expr.Call#,\n" - " // Result\n" - " _==_(\n" - " __result__^#14:Expr.Ident#,\n" - " 1^#15:int64#\n" - " )^#16:Expr.Call#)^#17:Expr.Comprehension#"}, - {"[].all(x, x > 0)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " true^#7:bool#,\n" - " // LoopCondition\n" - " @not_strictly_false(\n" - " __result__^#8:Expr.Ident#\n" - " )^#9:Expr.Call#,\n" - " // LoopStep\n" - " _&&_(\n" - " __result__^#10:Expr.Ident#,\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#\n" - " )^#11:Expr.Call#,\n" - " // Result\n" - " __result__^#12:Expr.Ident#)^#13:Expr.Comprehension#"}, - {"[].map(x, x + 1)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " []^#7:Expr.CreateList#,\n" - " // LoopCondition\n" - " true^#8:bool#,\n" - " // LoopStep\n" - " _+_(\n" - " __result__^#9:Expr.Ident#,\n" - " [\n" - " _+_(\n" - " x^#4:Expr.Ident#,\n" - " 1^#6:int64#\n" - " )^#5:Expr.Call#\n" - " ]^#10:Expr.CreateList#\n" - " )^#11:Expr.Call#,\n" - " // Result\n" - " __result__^#12:Expr.Ident#)^#13:Expr.Comprehension#"}, - {"[].map(x, x > 0, x + 1)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " []^#10:Expr.CreateList#,\n" - " // LoopCondition\n" - " true^#11:bool#,\n" - " // LoopStep\n" - " _?_:_(\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#,\n" - " _+_(\n" - " __result__^#12:Expr.Ident#,\n" - " [\n" - " _+_(\n" - " x^#7:Expr.Ident#,\n" - " 1^#9:int64#\n" - " )^#8:Expr.Call#\n" - " ]^#13:Expr.CreateList#\n" - " )^#14:Expr.Call#,\n" - " __result__^#15:Expr.Ident#\n" - " )^#16:Expr.Call#,\n" - " // Result\n" - " __result__^#17:Expr.Ident#)^#18:Expr.Comprehension#"}, - {"[].filter(x, x > 0)", - "__comprehension__(\n" - " // Variable\n" - " x,\n" - " // Target\n" - " []^#1:Expr.CreateList#,\n" - " // Accumulator\n" - " __result__,\n" - " // Init\n" - " []^#7:Expr.CreateList#,\n" - " // LoopCondition\n" - " true^#8:bool#,\n" - " // LoopStep\n" - " _?_:_(\n" - " _>_(\n" - " x^#4:Expr.Ident#,\n" - " 0^#6:int64#\n" - " )^#5:Expr.Call#,\n" - " _+_(\n" - " __result__^#9:Expr.Ident#,\n" - " [\n" - " x^#3:Expr.Ident#\n" - " ]^#10:Expr.CreateList#\n" - " )^#11:Expr.Call#,\n" - " __result__^#12:Expr.Ident#\n" - " )^#13:Expr.Call#,\n" - " // Result\n" - " __result__^#14:Expr.Ident#)^#15:Expr.Comprehension#"}, - // Maintain restriction on '__result__' variable name until the default is - // changed everywhere. - { - "[].map(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:20: map() variable name cannot be __result__\n" - " | [].map(__result__, true)\n" - " | ...................^", - }, - { - "[].map(__result__, true, false)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:20: map() variable name cannot be __result__\n" - " | [].map(__result__, true, false)\n" - " | ...................^", - }, - { - "[].filter(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:23: filter() variable name cannot be __result__\n" - " | [].filter(__result__, true)\n" - " | ......................^", - }, - { - "[].exists(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:23: exists() variable name cannot be __result__\n" - " | [].exists(__result__, true)\n" - " | ......................^", - }, - { - "[].all(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:20: all() variable name cannot be __result__\n" - " | [].all(__result__, true)\n" - " | ...................^", - }, - { - "[].exists_one(__result__, true)", - /*.P=*/"", - /*.E=*/ - "ERROR: :1:27: exists_one() variable name cannot be " - "__result__\n" - " | [].exists_one(__result__, true)\n" - " | ..........................^", - }}; - return *kInstance; -} - -class UpdatedAccuVarDisabledTest : public testing::TestWithParam {}; - -TEST_P(UpdatedAccuVarDisabledTest, Parse) { - const TestInfo& test_info = GetParam(); - ParserOptions options; - options.enable_hidden_accumulator_var = false; - if (!test_info.M.empty()) { - options.add_macro_calls = true; - } - - auto result = - EnrichedParse(test_info.I, Macro::AllMacros(), "", options); - if (test_info.E.empty()) { - EXPECT_THAT(result, IsOk()); - } else { - EXPECT_THAT(result, Not(IsOk())); - EXPECT_EQ(test_info.E, result.status().message()); - } - - if (!test_info.P.empty()) { - KindAndIdAdorner kind_and_id_adorner; - ExprPrinter w(kind_and_id_adorner); - std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); - EXPECT_EQ(test_info.P, adorned_string) - << result->parsed_expr().ShortDebugString(); - } - - if (!test_info.L.empty()) { - LocationAdorner location_adorner(result->parsed_expr().source_info()); - ExprPrinter w(location_adorner); - std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); - EXPECT_EQ(test_info.L, adorned_string) - << result->parsed_expr().ShortDebugString(); - } - - if (!test_info.R.empty()) { - EXPECT_EQ(test_info.R, ConvertEnrichedSourceInfoToString( - result->enriched_source_info())); - } - - if (!test_info.M.empty()) { - EXPECT_EQ(test_info.M, ConvertMacroCallsToString( - result.value().parsed_expr().source_info())) - << result->parsed_expr().ShortDebugString(); - } -} - TEST(NewParserBuilderTest, Defaults) { auto builder = cel::NewParserBuilder(); ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); @@ -2058,9 +1792,5 @@ std::string TestName(const testing::TestParamInfo& test_info) { INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionTest, testing::ValuesIn(test_cases), TestName); -INSTANTIATE_TEST_SUITE_P(UpdatedAccuVarTest, UpdatedAccuVarDisabledTest, - testing::ValuesIn(UpdatedAccuVarTestCases()), - TestName); - } // namespace } // namespace google::api::expr::parser From 1cf21eec91baa4181b481ff4bf6e25b9b5e9afe9 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 4 May 2026 10:35:24 -0700 Subject: [PATCH 003/350] Optionally track structured error messages in parser. Add a parse overload that reports issues with location to an output parameter where reasonable. This is used make the error handling more consistent when using bundled parse + typecheck. PiperOrigin-RevId: 910112013 --- compiler/BUILD | 2 + compiler/compiler.h | 2 + compiler/compiler_factory.cc | 49 +++++++++++++++----- compiler/compiler_factory_test.cc | 14 ++++++ extensions/math_ext_test.cc | 38 +++++----------- parser/BUILD | 2 + parser/parser.cc | 74 +++++++++++++++++++++---------- parser/parser_interface.h | 50 ++++++++++++++++++++- parser/parser_test.cc | 19 ++++++++ 9 files changed, 186 insertions(+), 64 deletions(-) diff --git a/compiler/BUILD b/compiler/BUILD index 50bc1c9fa..d4a0ab4ac 100644 --- a/compiler/BUILD +++ b/compiler/BUILD @@ -42,10 +42,12 @@ cc_library( hdrs = ["compiler_factory.h"], deps = [ ":compiler", + "//checker:type_check_issue", "//checker:type_checker", "//checker:type_checker_builder", "//checker:type_checker_builder_factory", "//checker:validation_result", + "//common:ast", "//common:source", "//internal:noop_delete", "//internal:status_macros", diff --git a/compiler/compiler.h b/compiler/compiler.h index 6d07e72c2..27237df60 100644 --- a/compiler/compiler.h +++ b/compiler/compiler.h @@ -97,6 +97,8 @@ struct CompilerLibrarySubset { struct CompilerOptions { ParserOptions parser_options; CheckerOptions checker_options; + // If true, parse errors will be adapted to issues where possible. + bool adapt_parser_errors = false; }; // Interface for CEL CompilerBuilder objects. diff --git a/compiler/compiler_factory.cc b/compiler/compiler_factory.cc index 14586825e..ed22c5630 100644 --- a/compiler/compiler_factory.cc +++ b/compiler/compiler_factory.cc @@ -17,16 +17,19 @@ #include #include #include +#include #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "checker/type_check_issue.h" #include "checker/type_checker.h" #include "checker/type_checker_builder.h" #include "checker/type_checker_builder_factory.h" #include "checker/validation_result.h" +#include "common/ast.h" #include "common/source.h" #include "compiler/compiler.h" #include "internal/status_macros.h" @@ -45,19 +48,38 @@ class CompilerImpl : public Compiler { CompilerImpl(std::unique_ptr type_checker, std::unique_ptr parser, // Copy the validator in case builder is reused. - Validator validator) + Validator validator, CompilerOptions options) : type_checker_(std::move(type_checker)), parser_(std::move(parser)), - validator_(std::move(validator)) {} + validator_(std::move(validator)), + options_(options) {} absl::StatusOr Compile( absl::string_view expression, absl::string_view description, google::protobuf::Arena* arena) const override { CEL_ASSIGN_OR_RETURN(auto source, cel::NewSource(expression, std::string(description))); - CEL_ASSIGN_OR_RETURN(auto ast, parser_->Parse(*source)); + std::vector parse_issues; + absl::StatusOr> ast = + parser_->Parse(*source, &parse_issues); + if (!ast.ok()) { + if (!options_.adapt_parser_errors || + ast.status().code() != absl::StatusCode::kInvalidArgument || + parse_issues.empty()) { + return ast.status(); + } + std::vector check_issues; + check_issues.reserve(parse_issues.size()); + for (const auto& issue : parse_issues) { + check_issues.push_back(TypeCheckIssue::CreateError( + issue.location(), std::string(issue.message()))); + } + ValidationResult result(std::move(check_issues)); + result.SetSource(std::move(source)); + return result; + } CEL_ASSIGN_OR_RETURN(ValidationResult result, - type_checker_->Check(std::move(ast), arena)); + type_checker_->Check(*std::move(ast), arena)); result.SetSource(std::move(source)); if (!validator_.validations().empty()) { @@ -76,16 +98,18 @@ class CompilerImpl : public Compiler { std::unique_ptr type_checker_; std::unique_ptr parser_; Validator validator_; + CompilerOptions options_; }; class CompilerBuilderImpl : public CompilerBuilder { public: CompilerBuilderImpl(std::unique_ptr type_checker_builder, std::unique_ptr parser_builder, - Validator validator = Validator()) + Validator validator, CompilerOptions options) : type_checker_builder_(std::move(type_checker_builder)), parser_builder_(std::move(parser_builder)), - validator_(std::move(validator)) {} + validator_(std::move(validator)), + options_(options) {} absl::Status AddLibrary(CompilerLibrary library) override { if (!library.id.empty()) { @@ -146,23 +170,23 @@ class CompilerBuilderImpl : public CompilerBuilder { absl::StatusOr> Build() override { CEL_ASSIGN_OR_RETURN(auto parser, parser_builder_->Build()); CEL_ASSIGN_OR_RETURN(auto type_checker, type_checker_builder_->Build()); - return std::make_unique(std::move(type_checker), - std::move(parser), validator_); + return std::make_unique( + std::move(type_checker), std::move(parser), validator_, options_); } private: std::unique_ptr type_checker_builder_; std::unique_ptr parser_builder_; Validator validator_; + CompilerOptions options_; absl::flat_hash_set library_ids_; absl::flat_hash_set subsets_; }; std::unique_ptr CompilerImpl::ToBuilder() const { - auto builder = std::make_unique( - type_checker_->ToBuilder(), parser_->ToBuilder(), validator_); - return builder; + return std::make_unique( + type_checker_->ToBuilder(), parser_->ToBuilder(), validator_, options_); } } // namespace @@ -179,7 +203,8 @@ absl::StatusOr> NewCompilerBuilder( auto parser_builder = NewParserBuilder(options.parser_options); return std::make_unique(std::move(type_checker_builder), - std::move(parser_builder)); + std::move(parser_builder), + Validator(), options); } } // namespace cel diff --git a/compiler/compiler_factory_test.cc b/compiler/compiler_factory_test.cc index 214c23765..035fd8aa6 100644 --- a/compiler/compiler_factory_test.cc +++ b/compiler/compiler_factory_test.cc @@ -413,5 +413,19 @@ TEST(CompilerFactoryTest, SpecifyArenaKeepsResolvedTypes) { it->second.GetOptional().GetParameter().GetList().GetElement().IsInt()); } +TEST(CompilerFactoryTest, ReturnsIssuesFromParser) { + CompilerOptions opts; + opts.adapt_parser_errors = true; + ASSERT_OK_AND_ASSIGN( + auto builder, NewCompilerBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, compiler->Compile("a +")); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), testing::Not(testing::IsEmpty())); +} + } // namespace } // namespace cel diff --git a/extensions/math_ext_test.cc b/extensions/math_ext_test.cc index 3088e6fa8..72605648f 100644 --- a/extensions/math_ext_test.cc +++ b/extensions/math_ext_test.cc @@ -23,7 +23,6 @@ #include "absl/algorithm/container.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" @@ -110,19 +109,6 @@ struct MacroTestCase { absl::string_view err = ""; }; -std::string FormatIssues(const cel::ValidationResult& result) { - std::string issues; - for (const auto& issue : result.GetIssues()) { - if (!issues.empty()) { - absl::StrAppend(&issues, "\n", - issue.ToDisplayString(*result.GetSource())); - } else { - issues = issue.ToDisplayString(*result.GetSource()); - } - } - return issues; -} - class TestFunction : public CelFunction { public: explicit TestFunction(absl::string_view name) @@ -352,10 +338,11 @@ TEST_P(MathExtMacroParamsTest, ParserTests) { TEST_P(MathExtMacroParamsTest, ParserAndCheckerTests) { const MacroTestCase& test_case = GetParam(); - - ASSERT_OK_AND_ASSIGN( - auto compiler_builder, - cel::NewCompilerBuilder(internal::GetTestingDescriptorPool())); + CompilerOptions compile_opts; + compile_opts.adapt_parser_errors = true; + ASSERT_OK_AND_ASSIGN(auto compiler_builder, + cel::NewCompilerBuilder( + internal::GetTestingDescriptorPool(), compile_opts)); ASSERT_THAT(compiler_builder->AddLibrary(StandardCheckerLibrary()), IsOk()); ASSERT_THAT(compiler_builder->AddLibrary(MathCompilerLibrary()), IsOk()); @@ -381,16 +368,16 @@ TEST_P(MathExtMacroParamsTest, ParserAndCheckerTests) { ASSERT_OK_AND_ASSIGN(auto compiler, std::move(*compiler_builder).Build()); - auto result = compiler->Compile(test_case.expr, ""); + ASSERT_OK_AND_ASSIGN(auto result, + compiler->Compile(test_case.expr, "")); if (!test_case.err.empty()) { - EXPECT_THAT(result.status(), StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr(test_case.err))); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatError(), HasSubstr(test_case.err)); return; } - ASSERT_THAT(result, IsOk()); - ASSERT_TRUE(result->IsValid()) << FormatIssues(*result); + ASSERT_TRUE(result.IsValid()) << result.FormatError(); RuntimeOptions opts; ASSERT_OK_AND_ASSIGN( @@ -411,9 +398,8 @@ TEST_P(MathExtMacroParamsTest, ParserAndCheckerTests) { IsOk()); ASSERT_OK_AND_ASSIGN(auto runtime, std::move(runtime_builder).Build()); - - ASSERT_OK_AND_ASSIGN(auto program, - runtime->CreateProgram(*result->ReleaseAst())); + ASSERT_OK_AND_ASSIGN(auto ast, result.ReleaseAst()); + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; cel::Activation activation; diff --git a/parser/BUILD b/parser/BUILD index 63813bb59..6650d9fe9 100644 --- a/parser/BUILD +++ b/parser/BUILD @@ -244,9 +244,11 @@ cc_library( ":options", "//common:ast", "//common:source", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", ], ) diff --git a/parser/parser.cc b/parser/parser.cc index f4ee3a1c5..709e2fd41 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -112,13 +112,12 @@ struct ParserError { }; std::string DisplayParserError(const cel::Source& source, - const ParserError& error) { - auto location = - source.GetLocation(error.range.begin).value_or(SourceLocation{}); + SourceLocation location, + absl::string_view message) { return absl::StrCat(absl::StrFormat("ERROR: %s:%zu:%zu: %s", source.description(), location.line, // add one to the 0-based column - location.column + 1, error.message), + location.column + 1, message), source.DisplayErrorLocation(location)); } @@ -209,7 +208,7 @@ class ParserMacroExprFactory final : public MacroExprFactory { bool HasErrors() const { return error_count_ != 0; } - std::string ErrorMessage() { + std::vector CollectIssues() { // Errors are collected as they are encountered, not by their location // within the source. To have a more stable error message as implementation // details change, we sort the collected errors by their source location @@ -226,20 +225,23 @@ class ParserMacroExprFactory final : public MacroExprFactory { }); // Build the summary error message using the sorted errors. bool errors_truncated = error_count_ > 100; - std::vector messages; - messages.reserve( + std::vector issues; + issues.reserve( errors_.size() + errors_truncated); // Reserve space for the transform and an // additional element when truncation occurs. - std::transform(errors_.begin(), errors_.end(), std::back_inserter(messages), - [this](const ParserError& error) { - return cel::DisplayParserError(source_, error); - }); + std::transform( + errors_.begin(), errors_.end(), std::back_inserter(issues), + [this](const ParserError& error) { + auto location = + source_.GetLocation(error.range.begin).value_or(SourceLocation{}); + return cel::ParseIssue(location, error.message); + }); if (errors_truncated) { - messages.emplace_back( - absl::StrCat(error_count_ - 100, " more errors were truncated.")); + issues.push_back(cel::ParseIssue( + absl::StrCat(error_count_ - 100, " more errors were truncated."))); } - return absl::StrJoin(messages, "\n"); + return issues; } void AddMacroCall(int64_t macro_id, absl::string_view function, @@ -602,6 +604,15 @@ Expr ExpressionBalancer::BalancedTree(int lo, int hi) { return factory_.NewCall(ops_[mid], function_, std::move(arguments)); } +std::string FormatIssues(const cel::Source& source, + absl::Span issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend(out, cel::DisplayParserError(source, issue.location(), + issue.message())); + }); +} + class ParserVisitor final : public CelBaseVisitor, public antlr4::BaseErrorListener { public: @@ -673,7 +684,7 @@ class ParserVisitor final : public CelBaseVisitor, const std::string& msg, std::exception_ptr e) override; bool HasErrored() const; - std::string ErrorMessage(); + std::vector CollectIssues(); private: template @@ -1434,7 +1445,9 @@ void ParserVisitor::syntaxError(antlr4::Recognizer* recognizer, bool ParserVisitor::HasErrored() const { return factory_.HasErrors(); } -std::string ParserVisitor::ErrorMessage() { return factory_.ErrorMessage(); } +std::vector ParserVisitor::CollectIssues() { + return factory_.CollectIssues(); +} Expr ParserVisitor::GlobalCallOrMacroImpl(int64_t expr_id, absl::string_view function, @@ -1638,9 +1651,10 @@ struct ParseResult { EnrichedSourceInfo enriched_source_info; }; -absl::StatusOr ParseImpl(const cel::Source& source, - const cel::MacroRegistry& registry, - const ParserOptions& options) { +absl::StatusOr ParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, + std::vector* parse_issues = nullptr) { try { CodePointStream input(source.content(), source.description()); if (input.size() > options.expression_size_codepoint_limit) { @@ -1673,13 +1687,23 @@ absl::StatusOr ParseImpl(const cel::Source& source, expr = ExprFromAny(visitor.visit(parser.start())); } catch (const ParseCancellationException& e) { if (visitor.HasErrored()) { - return absl::InvalidArgumentError(visitor.ErrorMessage()); + auto issues = visitor.CollectIssues(); + std::string error_message = FormatIssues(source, issues); + if (parse_issues != nullptr) { + *parse_issues = std::move(issues); + } + return absl::InvalidArgumentError(error_message); } return absl::CancelledError(e.what()); } if (visitor.HasErrored()) { - return absl::InvalidArgumentError(visitor.ErrorMessage()); + auto issues = visitor.CollectIssues(); + std::string error_message = FormatIssues(source, issues); + if (parse_issues != nullptr) { + *parse_issues = std::move(issues); + } + return absl::InvalidArgumentError(error_message); } return { @@ -1706,10 +1730,12 @@ class ParserImpl : public cel::Parser { macro_registry_(std::move(macro_registry)), library_ids_(std::move(library_ids)) {} - absl::StatusOr> Parse( - const cel::Source& source) const override { + absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* parse_issues) const override { CEL_ASSIGN_OR_RETURN(auto parse_result, - ParseImpl(source, macro_registry_, options_)); + ::google::api::expr::parser::ParseImpl( + source, macro_registry_, options_, parse_issues)); return std::make_unique(std::move(parse_result.expr), std::move(parse_result.source_info)); } diff --git a/parser/parser_interface.h b/parser/parser_interface.h index 7cc21ff26..ad6e8ca84 100644 --- a/parser/parser_interface.h +++ b/parser/parser_interface.h @@ -16,10 +16,14 @@ #include #include +#include +#include +#include "absl/base/nullability.h" #include "absl/functional/any_invocable.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "common/ast.h" #include "common/source.h" #include "parser/macro.h" @@ -73,6 +77,26 @@ class ParserBuilder { virtual absl::StatusOr> Build() = 0; }; +// Information about a parse failure. +class ParseIssue { + public: + explicit ParseIssue(std::string message) : message_(std::move(message)) {} + ParseIssue(SourceLocation location, std::string message) + : location_(location), message_(std::move(message)) {} + + ParseIssue(const ParseIssue& other) = default; + ParseIssue& operator=(const ParseIssue& other) = default; + ParseIssue(ParseIssue&& other) = default; + ParseIssue& operator=(ParseIssue&& other) = default; + + SourceLocation location() const { return location_; } + absl::string_view message() const { return message_; } + + private: + SourceLocation location_; + std::string message_; +}; + // Interface for stateful CEL parser objects for use with a `Compiler` // (bundled parse and type check). This is not needed for most users: // prefer using the free functions in `parser.h` for more flexibility. @@ -81,13 +105,35 @@ class Parser { virtual ~Parser() = default; // Parses the given source into a CEL AST. - virtual absl::StatusOr> Parse( - const cel::Source& source) const = 0; + absl::StatusOr> Parse( + const cel::Source& source) const; + + // Parses the given source into a CEL AST, collecting parse errors in + // `issues`. If `issues` is non-null, it will be cleared and all parse + // issues will be appended to it. + absl::StatusOr> Parse( + const cel::Source& source, std::vector* issues) const; // Returns a builder initialized with the configuration of this parser. virtual std::unique_ptr ToBuilder() const = 0; + + protected: + virtual absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const = 0; }; +inline absl::StatusOr> Parser::Parse( + const cel::Source& source) const { + return ParseImpl(source, nullptr); +} + +inline absl::StatusOr> Parser::Parse( + const cel::Source& source, std::vector* issues) const { + if (issues != nullptr) issues->clear(); + return ParseImpl(source, issues); +} + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_PARSER_PARSER_INTERFACE_H_ diff --git a/parser/parser_test.cc b/parser/parser_test.cc index a1a65481d..587b63a30 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1782,6 +1782,25 @@ TEST(NewParserBuilderTest, ToBuilderPreservesStdlibAndOptionalFromOptions) { EXPECT_FALSE(ast->IsChecked()); } +TEST(ParserTest, ParseFailurePopulatesIssues) { + auto builder = cel::NewParserBuilder(); + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("a +", "test.cel")); + std::vector issues; + auto ast_result = parser->Parse(*source, &issues); + EXPECT_THAT(ast_result, Not(IsOk())); + ASSERT_THAT(issues, testing::SizeIs(1)); + EXPECT_THAT(ast_result.status().message(), + HasSubstr("ERROR: test.cel:1:4: Syntax error: mismatched input " + "'' expecting")); + EXPECT_THAT(issues[0].message(), + HasSubstr("Syntax error: mismatched input '' expecting")); + EXPECT_EQ(issues[0].location().line, 1); + // 0-based, but adjusted to 1-based in error message. + EXPECT_EQ(issues[0].location().column, 3); +} + std::string TestName(const testing::TestParamInfo& test_info) { std::string name = absl::StrCat(test_info.index, "-", test_info.param.I); absl::c_replace_if(name, [](char c) { return !absl::ascii_isalnum(c); }, '_'); From 2f06d90f5b593269c2b1f58de3bfd5c8fc2fa895 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 4 May 2026 12:32:22 -0700 Subject: [PATCH 004/350] Add checker support for block. This is needed for re-checking expressions that were produced as a part of policy compilation. PiperOrigin-RevId: 910179322 --- checker/internal/BUILD | 6 + checker/internal/type_checker_impl.cc | 77 ++++++++- checker/internal/type_checker_impl_test.cc | 95 +++++++++++ conformance/BUILD | 5 +- conformance/service.cc | 115 +------------- extensions/BUILD | 5 +- extensions/bindings_ext.cc | 32 +++- extensions/bindings_ext.h | 6 +- testutil/BUILD | 20 +++ testutil/test_macros.cc | 175 +++++++++++++++++++++ testutil/test_macros.h | 33 ++++ 11 files changed, 446 insertions(+), 123 deletions(-) create mode 100644 testutil/test_macros.cc create mode 100644 testutil/test_macros.h diff --git a/checker/internal/BUILD b/checker/internal/BUILD index 1c560cdb9..f4c60f937 100644 --- a/checker/internal/BUILD +++ b/checker/internal/BUILD @@ -155,6 +155,7 @@ cc_library( "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -179,6 +180,7 @@ cc_test( "//checker:type_checker_builder", "//checker:validation_result", "//common:ast", + "//common:ast_proto", "//common:container", "//common:decl", "//common:expr", @@ -187,13 +189,17 @@ cc_test( "//internal:status_macros", "//internal:testing", "//internal:testing_descriptor_pool", + "//parser", + "//parser:macro_registry", "//testutil:baseline_tests", + "//testutil:test_macros", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 05601fdbb..2472d7def 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -25,6 +25,7 @@ #include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" @@ -59,6 +60,15 @@ namespace cel::checker_internal { namespace { +bool MatchesBlock(const Expr& expr) { + if (!expr.has_call_expr()) { + return false; + } + const auto& call = expr.call_expr(); + return call.function() == "cel.@block" && call.args().size() == 2 && + call.args()[0].has_list_expr(); +} + using AstType = cel::TypeSpec; using Severity = TypeCheckIssue::Severity; @@ -204,13 +214,23 @@ class ResolveVisitor : public AstVisitorBase { arena_(arena), current_scope_(&root_scope_) {} - void PreVisitExpr(const Expr& expr) override { expr_stack_.push_back(&expr); } + void PreVisitExpr(const Expr& expr) override { + expr_stack_.push_back(&expr); + if (expr_stack_.size() == 1 && MatchesBlock(expr)) { + ABSL_DCHECK_EQ(expr.call_expr().args().size(), 2); + ABSL_DCHECK(block_init_list_ == nullptr); + block_init_list_ = &expr.call_expr().args()[0]; + } + } void PostVisitExpr(const Expr& expr) override { if (expr_stack_.empty()) { return; } expr_stack_.pop_back(); + if (expr_stack_.size() == 2 && expr_stack_.back() == block_init_list_) { + HandleBlockIndex(&expr); + } } void PostVisitConst(const Expr& expr, const Constant& constant) override; @@ -389,6 +409,7 @@ class ResolveVisitor : public AstVisitorBase { absl::string_view field_name); void HandleOptSelect(const Expr& expr); + void HandleBlockIndex(const Expr* expr); // Get the assigned type of the given subexpression. Should only be called if // the given subexpression is expected to have already been checked. @@ -421,6 +442,7 @@ class ResolveVisitor : public AstVisitorBase { std::vector expr_stack_; absl::flat_hash_map> maybe_namespaced_functions_; + const Expr* block_init_list_ = nullptr; // Select operations that need to be resolved outside of the traversal. // These are handled separately to disambiguate between namespaces and field // accesses @@ -609,8 +631,15 @@ void ResolveVisitor::PostVisitMap(const Expr& expr, const MapExpr& map) { } void ResolveVisitor::PostVisitList(const Expr& expr, const ListExpr& list) { - // Follows list type inferencing behavior in Go (see map comments above). + if (&expr == block_init_list_) { + // Don't try to coalesce list type here because it can influence the + // resolved type of the list elements. cel.@block is always list and + // the elements are treated independently at runtime. + types_[&expr] = ListType(); + return; + } + // Follows list type inferencing behavior in Go (see map comments above). Type overall_elem_type = inference_context_->InstantiateTypeParams(TypeParamType("E")); auto assignability_context = inference_context_->CreateAssignabilityContext(); @@ -1172,6 +1201,44 @@ void ResolveVisitor::HandleOptSelect(const Expr& expr) { } } +void ResolveVisitor::HandleBlockIndex(const Expr* expr) { + ABSL_DCHECK(block_init_list_ != nullptr); + ABSL_DCHECK(block_init_list_->has_list_expr()); + const auto& elements = block_init_list_->list_expr().elements(); + int index = -1; + for (size_t i = 0; i < elements.size(); ++i) { + if (&elements[i].expr() == expr) { + index = i; + break; + } + } + if (index < 0) { + status_.Update(absl::InternalError( + "could not resolve expression as a cel.@block subexpression")); + return; + } + std::string var_name = absl::StrCat("@index", index); + + // Block is typically manually assembled from logically separate + // expressions so fix the type instead of inferring any remaining free type + // params as for normal subexpressions. + auto type = inference_context_->FinalizeType(GetDeducedType(expr)); + + VariableDecl decl = MakeVariableDecl(var_name, std::move(type)); + + // The C++ runtime requires that the indexes are topologically ordered. + // They just come into scope in order as we walk the AST so we don't need + // to do any additional work to check references to other initializers in + // an init expr. + // + // TODO(uncreated-issue/90): This is slightly inconsistent with the java + // runtime implementation which just requires the references to be acyclic. + auto* scope = + comprehension_vars_.emplace_back(current_scope_->MakeNestedScope()).get(); + scope->InsertVariableIfAbsent(std::move(decl)); + current_scope_ = scope; +} + class ResolveRewriter : public AstRewriterBase { public: explicit ResolveRewriter(const ResolveVisitor& visitor, @@ -1230,15 +1297,15 @@ class ResolveRewriter : public AstRewriterBase { if (auto iter = visitor_.types().find(&expr); iter != visitor_.types().end()) { - auto flattened_type = - FlattenType(inference_context_.FinalizeType(iter->second)); + cel::Type finalized_type = inference_context_.FinalizeType(iter->second); + auto flattened_type = FlattenType(finalized_type); if (!flattened_type.ok()) { status_.Update(flattened_type.status()); return rewritten; } type_map_[expr.id()] = *std::move(flattened_type); - resolved_types_[expr.id()] = iter->second; + resolved_types_[expr.id()] = finalized_type; rewritten = true; } diff --git a/checker/internal/type_checker_impl_test.cc b/checker/internal/type_checker_impl_test.cc index e6cd641d6..893f0689d 100644 --- a/checker/internal/type_checker_impl_test.cc +++ b/checker/internal/type_checker_impl_test.cc @@ -26,6 +26,7 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -36,6 +37,7 @@ #include "checker/type_checker_builder.h" #include "checker/validation_result.h" #include "common/ast.h" +#include "common/ast_proto.h" #include "common/container.h" #include "common/decl.h" #include "common/expr.h" @@ -45,7 +47,10 @@ #include "internal/status_macros.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" +#include "parser/macro_registry.h" +#include "parser/parser.h" #include "testutil/baseline_tests.h" +#include "testutil/test_macros.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" @@ -108,6 +113,17 @@ google::protobuf::Arena* absl_nonnull TestTypeArena() { return &(*kArena); } +absl::StatusOr> MakeTestParsedAstWithMacros( + absl::string_view expression, const cel::MacroRegistry& registry) { + CEL_ASSIGN_OR_RETURN( + auto source, + cel::NewSource(expression, /*description=*/std::string(expression))); + CEL_ASSIGN_OR_RETURN(auto parsed_expr, google::api::expr::parser::Parse( + *source, registry, + {.enable_optional_syntax = true})); + return cel::CreateAstFromParsedExpr(parsed_expr); +} + FunctionDecl MakeIdentFunction() { auto decl = MakeFunctionDecl( "identity", @@ -272,6 +288,12 @@ absl::Status RegisterMinimalBuiltins(google::protobuf::Arena* absl_nonnull arena /*return_type=*/TypeType(arena, TypeParamType("A")), TypeParamType("A")))); + Type kParam(TypeParamType("T")); + CEL_ASSIGN_OR_RETURN( + auto block_decl, + MakeFunctionDecl("cel.@block", MakeOverloadDecl("cel_block_list", kParam, + ListType(), kParam))); + env.InsertFunctionIfAbsent(std::move(not_op)); env.InsertFunctionIfAbsent(std::move(not_strictly_false)); env.InsertFunctionIfAbsent(std::move(add_op)); @@ -289,6 +311,7 @@ absl::Status RegisterMinimalBuiltins(google::protobuf::Arena* absl_nonnull arena env.InsertFunctionIfAbsent(std::move(to_type)); env.InsertFunctionIfAbsent(std::move(to_duration)); env.InsertFunctionIfAbsent(std::move(to_timestamp)); + env.InsertFunctionIfAbsent(std::move(block_decl)); return absl::OkStatus(); } @@ -308,6 +331,78 @@ TEST(TypeCheckerImplTest, SmokeTest) { EXPECT_THAT(result.GetIssues(), IsEmpty()); } +TEST(TypeCheckerImplTest, BlockMacroSupport) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + MacroRegistry registry; + ASSERT_THAT(cel::test::RegisterTestMacros(registry), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN( + auto ast, + MakeTestParsedAstWithMacros( + "cel.block([1, 2], cel.index(0) + cel.index(1))", registry)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + // Overall type should be int. + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + auto root_id = checked_ast->root_expr().id(); + EXPECT_EQ(checked_ast->type_map().at(root_id).primitive(), + PrimitiveType::kInt64); +} + +TEST(TypeCheckerImplTest, BlockMacroSupportMixedTypes) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + MacroRegistry registry; + ASSERT_THAT(cel::test::RegisterTestMacros(registry), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN( + auto ast, MakeTestParsedAstWithMacros("cel.block([1, 'a'], cel.index(1))", + registry)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + // cel.index(1) refers to 'a' which is string. + // So overall type should be string. + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + auto root_id = checked_ast->root_expr().id(); + EXPECT_EQ(checked_ast->type_map().at(root_id).primitive(), + PrimitiveType::kString); +} + +TEST(TypeCheckerImplTest, BadIndex) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + MacroRegistry registry; + ASSERT_THAT(cel::test::RegisterTestMacros(registry), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN( + auto ast, MakeTestParsedAstWithMacros("cel.block([1, 'a'], cel.index(2))", + registry)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatError(), + HasSubstr("undeclared reference to '@index2' (in container")); +} + TEST(TypeCheckerImplTest, SimpleIdentsResolved) { TypeCheckEnv env(GetSharedTestingDescriptorPool()); diff --git a/conformance/BUILD b/conformance/BUILD index 139739891..9b527cf35 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -69,6 +69,7 @@ cc_library( "//runtime:reference_resolver", "//runtime:runtime_options", "//runtime:standard_runtime_builder_factory", + "//testutil:test_macros", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", @@ -221,7 +222,7 @@ _TESTS_TO_SKIP_LEGACY = _TESTS_TO_SKIP + [ "proto3/set_null/list_value", "proto3/set_null/single_struct", - # cel.@block + # no optional support for legacy types "block_ext/basic/optional_list", "block_ext/basic/optional_map", "block_ext/basic/optional_map_chained", @@ -231,7 +232,7 @@ _TESTS_TO_SKIP_LEGACY = _TESTS_TO_SKIP + [ _TESTS_TO_SKIP_CHECKED = [ # block is a post-check optimization that inserts internal variables. The C++ type checker # needs support for a proper optimizer for this to work. - "block_ext", + # "block_ext", ] _TESTS_TO_SKIP_LEGACY_DASHBOARD = [ diff --git a/conformance/service.cc b/conformance/service.cc index 3edc214e6..463334bb5 100644 --- a/conformance/service.cc +++ b/conformance/service.cc @@ -14,7 +14,6 @@ #include "conformance/service.h" -#include #include #include #include @@ -36,11 +35,8 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" -#include "absl/types/optional.h" -#include "absl/types/span.h" #include "checker/optional.h" #include "checker/standard_library.h" #include "checker/type_checker_builder.h" @@ -48,7 +44,6 @@ #include "common/ast.h" #include "common/ast_proto.h" #include "common/decl_proto_v1alpha1.h" -#include "common/expr.h" #include "common/internal/value_conversion.h" #include "common/source.h" #include "common/value.h" @@ -72,8 +67,6 @@ #include "extensions/select_optimization.h" #include "extensions/strings.h" #include "internal/status_macros.h" -#include "parser/macro.h" -#include "parser/macro_expr_factory.h" #include "parser/macro_registry.h" #include "parser/options.h" #include "parser/parser.h" @@ -85,6 +78,7 @@ #include "runtime/runtime.h" #include "runtime/runtime_options.h" #include "runtime/standard_runtime_builder_factory.h" +#include "testutil/test_macros.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "cel/expr/conformance/proto2/test_all_types_extensions.pb.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" @@ -106,109 +100,6 @@ namespace google::api::expr::runtime { namespace { -bool IsCelNamespace(const cel::Expr& target) { - return target.has_ident_expr() && target.ident_expr().name() == "cel"; -} - -absl::optional CelBlockMacroExpander(cel::MacroExprFactory& factory, - cel::Expr& target, - absl::Span args) { - if (!IsCelNamespace(target)) { - return absl::nullopt; - } - cel::Expr& bindings_arg = args[0]; - if (!bindings_arg.has_list_expr()) { - return factory.ReportErrorAt( - bindings_arg, "cel.block requires the first arg to be a list literal"); - } - return factory.NewCall("cel.@block", args); -} - -absl::optional CelIndexMacroExpander(cel::MacroExprFactory& factory, - cel::Expr& target, - absl::Span args) { - if (!IsCelNamespace(target)) { - return absl::nullopt; - } - cel::Expr& index_arg = args[0]; - if (!index_arg.has_const_expr() || !index_arg.const_expr().has_int_value()) { - return factory.ReportErrorAt( - index_arg, "cel.index requires a single non-negative int constant arg"); - } - int64_t index = index_arg.const_expr().int_value(); - if (index < 0) { - return factory.ReportErrorAt( - index_arg, "cel.index requires a single non-negative int constant arg"); - } - return factory.NewIdent(absl::StrCat("@index", index)); -} - -absl::optional CelIterVarMacroExpander( - cel::MacroExprFactory& factory, cel::Expr& target, - absl::Span args) { - if (!IsCelNamespace(target)) { - return absl::nullopt; - } - cel::Expr& depth_arg = args[0]; - if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || - depth_arg.const_expr().int_value() < 0) { - return factory.ReportErrorAt( - depth_arg, "cel.iterVar requires two non-negative int constant args"); - } - cel::Expr& unique_arg = args[1]; - if (!unique_arg.has_const_expr() || - !unique_arg.const_expr().has_int_value() || - unique_arg.const_expr().int_value() < 0) { - return factory.ReportErrorAt( - unique_arg, "cel.iterVar requires two non-negative int constant args"); - } - return factory.NewIdent( - absl::StrCat("@it:", depth_arg.const_expr().int_value(), ":", - unique_arg.const_expr().int_value())); -} - -absl::optional CelAccuVarMacroExpander( - cel::MacroExprFactory& factory, cel::Expr& target, - absl::Span args) { - if (!IsCelNamespace(target)) { - return absl::nullopt; - } - cel::Expr& depth_arg = args[0]; - if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || - depth_arg.const_expr().int_value() < 0) { - return factory.ReportErrorAt( - depth_arg, "cel.accuVar requires two non-negative int constant args"); - } - cel::Expr& unique_arg = args[1]; - if (!unique_arg.has_const_expr() || - !unique_arg.const_expr().has_int_value() || - unique_arg.const_expr().int_value() < 0) { - return factory.ReportErrorAt( - unique_arg, "cel.accuVar requires two non-negative int constant args"); - } - return factory.NewIdent( - absl::StrCat("@ac:", depth_arg.const_expr().int_value(), ":", - unique_arg.const_expr().int_value())); -} - -absl::Status RegisterCelBlockMacros(cel::MacroRegistry& registry) { - CEL_ASSIGN_OR_RETURN(auto block_macro, - cel::Macro::Receiver("block", 2, CelBlockMacroExpander)); - CEL_RETURN_IF_ERROR(registry.RegisterMacro(block_macro)); - CEL_ASSIGN_OR_RETURN(auto index_macro, - cel::Macro::Receiver("index", 1, CelIndexMacroExpander)); - CEL_RETURN_IF_ERROR(registry.RegisterMacro(index_macro)); - CEL_ASSIGN_OR_RETURN( - auto iter_var_macro, - cel::Macro::Receiver("iterVar", 2, CelIterVarMacroExpander)); - CEL_RETURN_IF_ERROR(registry.RegisterMacro(iter_var_macro)); - CEL_ASSIGN_OR_RETURN( - auto accu_var_macro, - cel::Macro::Receiver("accuVar", 2, CelAccuVarMacroExpander)); - CEL_RETURN_IF_ERROR(registry.RegisterMacro(accu_var_macro)); - return absl::OkStatus(); -} - google::rpc::Code ToGrpcCode(absl::StatusCode code) { return static_cast(code); } @@ -250,7 +141,7 @@ absl::Status LegacyParse(const conformance::v1alpha1::ParseRequest& request, CEL_RETURN_IF_ERROR(cel::extensions::RegisterBindingsMacros(macros, options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterMathMacros(macros, options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterProtoMacros(macros, options)); - CEL_RETURN_IF_ERROR(RegisterCelBlockMacros(macros)); + CEL_RETURN_IF_ERROR(cel::test::RegisterTestMacros(macros)); CEL_ASSIGN_OR_RETURN(auto source, cel::NewSource(request.cel_source(), request.source_location())); CEL_ASSIGN_OR_RETURN(auto parsed_expr, @@ -285,6 +176,8 @@ absl::Status CheckImpl(google::protobuf::Arena* arena, if (!request.no_std_env()) { CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::StandardCheckerLibrary())); CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::OptionalCheckerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(cel::extensions::BindingsCheckerLibrary())); CEL_RETURN_IF_ERROR( builder->AddLibrary(cel::extensions::StringsCheckerLibrary())); CEL_RETURN_IF_ERROR( diff --git a/extensions/BUILD b/extensions/BUILD index ff37e2c3f..05104a4a5 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -215,7 +215,10 @@ cc_library( srcs = ["bindings_ext.cc"], hdrs = ["bindings_ext.h"], deps = [ - "//common:ast", + "//checker:type_checker_builder", + "//common:decl", + "//common:expr", + "//common:type", "//compiler", "//internal:status_macros", "//parser:macro", diff --git a/extensions/bindings_ext.cc b/extensions/bindings_ext.cc index f097709ca..c59f724bd 100644 --- a/extensions/bindings_ext.cc +++ b/extensions/bindings_ext.cc @@ -21,7 +21,10 @@ #include "absl/status/statusor.h" #include "absl/types/optional.h" #include "absl/types/span.h" -#include "common/ast.h" +#include "checker/type_checker_builder.h" +#include "common/decl.h" +#include "common/expr.h" +#include "common/type.h" #include "compiler/compiler.h" #include "internal/status_macros.h" #include "parser/macro.h" @@ -34,6 +37,8 @@ namespace { static constexpr char kCelNamespace[] = "cel"; static constexpr char kBind[] = "bind"; +static constexpr char kBlock[] = "cel.@block"; +static constexpr char kBlockOverloadId[] = "cel_block_list"; static constexpr char kUnusedIterVar[] = "#unused"; bool IsTargetNamespace(const Expr& target) { @@ -47,6 +52,19 @@ inline absl::Status ConfigureParser(ParserBuilder& parser_builder) { return absl::OkStatus(); } +absl::Status ConfigureChecker(int version, + TypeCheckerBuilder& type_checker_builder) { + if (version < 1) { + return absl::OkStatus(); + } + static Type kParam(TypeParamType("T")); + CEL_ASSIGN_OR_RETURN( + auto decl, + MakeFunctionDecl(kBlock, MakeOverloadDecl(kBlockOverloadId, kParam, + ListType(), kParam))); + return type_checker_builder.AddFunction(std::move(decl)); +} + } // namespace std::vector bindings_macros() { @@ -70,8 +88,16 @@ std::vector bindings_macros() { return {*cel_bind}; } -CompilerLibrary BindingsCompilerLibrary() { - return CompilerLibrary("cel.lib.ext.bindings", &ConfigureParser); +CompilerLibrary BindingsCompilerLibrary(int version) { + return CompilerLibrary( + "cel.lib.ext.bindings", &ConfigureParser, + [version](auto& b) { return ConfigureChecker(version, b); }); +} + +CheckerLibrary BindingsCheckerLibrary(int version) { + return CheckerLibrary{"cel.lib.ext.bindings", [version](auto& b) { + return ConfigureChecker(version, b); + }}; } } // namespace cel::extensions diff --git a/extensions/bindings_ext.h b/extensions/bindings_ext.h index a338b24f6..40b83a37f 100644 --- a/extensions/bindings_ext.h +++ b/extensions/bindings_ext.h @@ -25,6 +25,7 @@ namespace cel::extensions { +constexpr int kBindingsVersionLatest = 1; // bindings_macros() returns a macro for cel.bind() which can be used to support // local variable bindings within expressions. std::vector bindings_macros(); @@ -35,7 +36,10 @@ inline absl::Status RegisterBindingsMacros(MacroRegistry& registry, } // Declarations for the bindings extension library. -CompilerLibrary BindingsCompilerLibrary(); +CompilerLibrary BindingsCompilerLibrary(int version = kBindingsVersionLatest); + +// Declarations for the bindings extension library. +CheckerLibrary BindingsCheckerLibrary(int version = kBindingsVersionLatest); } // namespace cel::extensions diff --git a/testutil/BUILD b/testutil/BUILD index 292696033..782c95ca6 100644 --- a/testutil/BUILD +++ b/testutil/BUILD @@ -62,6 +62,26 @@ cc_library( deps = ["//internal:proto_matchers"], ) +cc_library( + name = "test_macros", + testonly = True, + srcs = ["test_macros.cc"], + hdrs = ["test_macros.h"], + deps = [ + "//common:expr", + "//internal:status_macros", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:macro_registry", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + ], +) + cc_library( name = "baseline_tests", testonly = True, diff --git a/testutil/test_macros.cc b/testutil/test_macros.cc new file mode 100644 index 000000000..158135762 --- /dev/null +++ b/testutil/test_macros.cc @@ -0,0 +1,175 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "testutil/test_macros.h" + +#include +#include + +#include "absl/base/no_destructor.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "common/expr.h" +#include "internal/status_macros.h" +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/macro_registry.h" + +namespace cel::test { + +namespace { + +bool IsCelNamespace(const Expr& target) { + return target.has_ident_expr() && target.ident_expr().name() == "cel"; +} + +absl::optional CelBlockMacroExpander(MacroExprFactory& factory, + Expr& target, + absl::Span args) { + if (!IsCelNamespace(target)) { + return absl::nullopt; + } + Expr& bindings_arg = args[0]; + if (!bindings_arg.has_list_expr()) { + return factory.ReportErrorAt( + bindings_arg, "cel.block requires the first arg to be a list literal"); + } + return factory.NewCall("cel.@block", args); +} + +absl::optional CelIndexMacroExpander(MacroExprFactory& factory, + Expr& target, + absl::Span args) { + if (!IsCelNamespace(target)) { + return absl::nullopt; + } + Expr& index_arg = args[0]; + if (!index_arg.has_const_expr() || !index_arg.const_expr().has_int_value()) { + return factory.ReportErrorAt( + index_arg, "cel.index requires a single non-negative int constant arg"); + } + int64_t index = index_arg.const_expr().int_value(); + if (index < 0) { + return factory.ReportErrorAt( + index_arg, "cel.index requires a single non-negative int constant arg"); + } + return factory.NewIdent(absl::StrCat("@index", index)); +} + +absl::optional CelIterVarMacroExpander(MacroExprFactory& factory, + Expr& target, + absl::Span args) { + if (!IsCelNamespace(target)) { + return absl::nullopt; + } + Expr& depth_arg = args[0]; + if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || + depth_arg.const_expr().int_value() < 0) { + return factory.ReportErrorAt( + depth_arg, "cel.iterVar requires two non-negative int constant args"); + } + Expr& unique_arg = args[1]; + if (!unique_arg.has_const_expr() || + !unique_arg.const_expr().has_int_value() || + unique_arg.const_expr().int_value() < 0) { + return factory.ReportErrorAt( + unique_arg, "cel.iterVar requires two non-negative int constant args"); + } + return factory.NewIdent( + absl::StrCat("@it:", depth_arg.const_expr().int_value(), ":", + unique_arg.const_expr().int_value())); +} + +absl::optional CelAccuVarMacroExpander(MacroExprFactory& factory, + Expr& target, + absl::Span args) { + if (!IsCelNamespace(target)) { + return absl::nullopt; + } + Expr& depth_arg = args[0]; + if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || + depth_arg.const_expr().int_value() < 0) { + return factory.ReportErrorAt( + depth_arg, "cel.accuVar requires two non-negative int constant args"); + } + Expr& unique_arg = args[1]; + if (!unique_arg.has_const_expr() || + !unique_arg.const_expr().has_int_value() || + unique_arg.const_expr().int_value() < 0) { + return factory.ReportErrorAt( + unique_arg, "cel.accuVar requires two non-negative int constant args"); + } + return factory.NewIdent( + absl::StrCat("@ac:", depth_arg.const_expr().int_value(), ":", + unique_arg.const_expr().int_value())); +} + +Macro MakeCelBlockMacro() { + auto macro_or_status = Macro::Receiver("block", 2, CelBlockMacroExpander); + ABSL_CHECK_OK(macro_or_status); // Crash OK + return std::move(*macro_or_status); +} + +Macro MakeCelIndexMacro() { + auto macro_or_status = Macro::Receiver("index", 1, CelIndexMacroExpander); + ABSL_CHECK_OK(macro_or_status); // Crash OK + return std::move(*macro_or_status); +} + +Macro MakeCelIterVarMacro() { + auto macro_or_status = Macro::Receiver("iterVar", 2, CelIterVarMacroExpander); + ABSL_CHECK_OK(macro_or_status); // Crash OK + return std::move(*macro_or_status); +} + +Macro MakeCelAccuVarMacro() { + auto macro_or_status = Macro::Receiver("accuVar", 2, CelAccuVarMacroExpander); + ABSL_CHECK_OK(macro_or_status); // Crash OK + return std::move(*macro_or_status); +} + +} // namespace + +const Macro& CelBlockMacro() { + static const absl::NoDestructor macro(MakeCelBlockMacro()); + return *macro; +} + +const Macro& CelIndexMacro() { + static const absl::NoDestructor macro(MakeCelIndexMacro()); + return *macro; +} + +const Macro& CelIterVarMacro() { + static const absl::NoDestructor macro(MakeCelIterVarMacro()); + return *macro; +} + +const Macro& CelAccuVarMacro() { + static const absl::NoDestructor macro(MakeCelAccuVarMacro()); + return *macro; +} + +absl::Status RegisterTestMacros(MacroRegistry& registry) { + CEL_RETURN_IF_ERROR(registry.RegisterMacro(CelBlockMacro())); + CEL_RETURN_IF_ERROR(registry.RegisterMacro(CelIndexMacro())); + CEL_RETURN_IF_ERROR(registry.RegisterMacro(CelIterVarMacro())); + CEL_RETURN_IF_ERROR(registry.RegisterMacro(CelAccuVarMacro())); + return absl::OkStatus(); +} + +} // namespace cel::test diff --git a/testutil/test_macros.h b/testutil/test_macros.h new file mode 100644 index 000000000..cad897999 --- /dev/null +++ b/testutil/test_macros.h @@ -0,0 +1,33 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_MACROS_H_ +#define THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_MACROS_H_ + +#include "absl/status/status.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" + +namespace cel::test { + +const Macro& CelBlockMacro(); +const Macro& CelIndexMacro(); +const Macro& CelIterVarMacro(); +const Macro& CelAccuVarMacro(); + +absl::Status RegisterTestMacros(MacroRegistry& registry); + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_MACROS_H_ From ddcece1479ed78ccde1594e47d94eeb841de115f Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 6 May 2026 13:56:21 -0700 Subject: [PATCH 005/350] Refactor optional dispatch tables. PiperOrigin-RevId: 911533283 --- common/values/optional_value.cc | 255 ++++++++++++++++---------------- 1 file changed, 124 insertions(+), 131 deletions(-) diff --git a/common/values/optional_value.cc b/common/values/optional_value.cc index ad0a65efb..688cf8fb0 100644 --- a/common/values/optional_value.cc +++ b/common/values/optional_value.cc @@ -122,200 +122,185 @@ absl::Status OptionalValueEqual( return absl::OkStatus(); } +google::protobuf::Arena* absl_nullable OptionalValueGetArenaNull( + const OpaqueValueDispatcher* absl_nonnull, OpaqueValueContent) { + return nullptr; +} + +OpaqueValue OptionalValueClone( + const OpaqueValueDispatcher* absl_nonnull dispatcher, + OpaqueValueContent content, google::protobuf::Arena* absl_nonnull arena) { + return common_internal::MakeOptionalValue(dispatcher, content); +} + +bool OptionalValueHasNoValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content) { + return false; +} + +void EmptyOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = + ErrorValue(absl::FailedPreconditionError("optional.none() dereference")); +} + +void NullOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = NullValue(); +} + +void BoolOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = BoolValue(content.To()); +} + +void IntOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = IntValue(content.To()); +} + +void UintOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = UintValue(content.To()); +} + +void DoubleOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = DoubleValue(content.To()); +} + +void DurationOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = UnsafeDurationValue(content.To()); +} + +void TimestampOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = UnsafeTimestampValue(content.To()); +} + ABSL_CONST_INIT const OptionalValueDispatcher empty_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) - -> google::protobuf::Arena* absl_nullable { return nullptr; }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, - }, - [](const OptionalValueDispatcher* absl_nonnull dispatcher, - CustomValueContent content) -> bool { return false; }, - [](const OptionalValueDispatcher* absl_nonnull dispatcher, - CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = ErrorValue( - absl::FailedPreconditionError("optional.none() dereference")); + .clone = &OptionalValueClone, }, + &OptionalValueHasNoValue, + &EmptyOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher null_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) -> google::protobuf::Arena* absl_nullable { - return nullptr; - }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent, - cel::Value* absl_nonnull result) -> void { *result = NullValue(); }, + &NullOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher bool_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) -> google::protobuf::Arena* absl_nullable { - return nullptr; - }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = BoolValue(content.To()); - }, + &BoolOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher int_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) -> google::protobuf::Arena* absl_nullable { - return nullptr; - }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = IntValue(content.To()); - }, + &IntOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher uint_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) -> google::protobuf::Arena* absl_nullable { - return nullptr; - }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = UintValue(content.To()); - }, + &UintOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher double_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) - -> google::protobuf::Arena* absl_nullable { return nullptr; }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, - CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = DoubleValue(content.To()); - }, + &DoubleOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher duration_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) - -> google::protobuf::Arena* absl_nullable { return nullptr; }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, - CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = UnsafeDurationValue(content.To()); - }, + &DurationOptionalValueValue, }; ABSL_CONST_INIT const OptionalValueDispatcher timestamp_optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent) - -> google::protobuf::Arena* absl_nullable { return nullptr; }, + .get_arena = &OptionalValueGetArenaNull, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - return common_internal::MakeOptionalValue(dispatcher, content); - }, + .clone = &OptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, - CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = UnsafeTimestampValue(content.To()); - }, + &TimestampOptionalValueValue, }; struct OptionalValueContent { @@ -323,43 +308,51 @@ struct OptionalValueContent { google::protobuf::Arena* absl_nonnull arena; }; +google::protobuf::Arena* absl_nullable GenericOptionalValueGetArena( + const OpaqueValueDispatcher* absl_nonnull, OpaqueValueContent content) { + return content.To().arena; +} + +OpaqueValue GenericOptionalValueClone( + const OpaqueValueDispatcher* absl_nonnull dispatcher, + OpaqueValueContent content, google::protobuf::Arena* absl_nonnull arena); + +void GenericOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, + CustomValueContent content, + cel::Value* absl_nonnull result) { + *result = *content.To().value; +} + ABSL_CONST_INIT const OptionalValueDispatcher optional_value_dispatcher = { { .get_type_id = &OptionalValueGetTypeId, - .get_arena = - [](const OpaqueValueDispatcher* absl_nonnull, - OpaqueValueContent content) -> google::protobuf::Arena* absl_nullable { - return content.To().arena; - }, + .get_arena = &GenericOptionalValueGetArena, .get_type_name = &OptionalValueGetTypeName, .debug_string = &OptionalValueDebugString, .get_runtime_type = &OptionalValueGetRuntimeType, .equal = &OptionalValueEqual, - .clone = [](const OpaqueValueDispatcher* absl_nonnull dispatcher, - OpaqueValueContent content, - google::protobuf::Arena* absl_nonnull arena) -> OpaqueValue { - ABSL_DCHECK(arena != nullptr); - - cel::Value* absl_nonnull result = ::new ( - arena->AllocateAligned(sizeof(cel::Value), alignof(cel::Value))) - cel::Value( - content.To().value->Clone(arena)); - if (!ArenaTraits<>::trivially_destructible(result)) { - arena->OwnDestructor(result); - } - return common_internal::MakeOptionalValue( - &optional_value_dispatcher, - OpaqueValueContent::From( - OptionalValueContent{.value = result, .arena = arena})); - }, + .clone = &GenericOptionalValueClone, }, &OptionalValueHasValue, - [](const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, - cel::Value* absl_nonnull result) -> void { - *result = *content.To().value; - }, + &GenericOptionalValueValue, }; +OpaqueValue GenericOptionalValueClone( + const OpaqueValueDispatcher* absl_nonnull dispatcher, + OpaqueValueContent content, google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + + cel::Value* absl_nonnull result = + ::new (arena->AllocateAligned(sizeof(cel::Value), alignof(cel::Value))) + cel::Value(content.To().value->Clone(arena)); + if (!ArenaTraits<>::trivially_destructible(result)) { + arena->OwnDestructor(result); + } + return common_internal::MakeOptionalValue( + &optional_value_dispatcher, OpaqueValueContent::From(OptionalValueContent{ + .value = result, .arena = arena})); +} + } // namespace OptionalValue OptionalValue::Of(cel::Value value, From 5806d30ba86ca40d8ab111e59fa78983afe5319c Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 8 May 2026 13:03:46 -0700 Subject: [PATCH 006/350] Update conformance test skip list PiperOrigin-RevId: 912656156 --- conformance/BUILD | 20 ++++++++++++++++++++ conformance/run.bzl | 6 +++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/conformance/BUILD b/conformance/BUILD index 9b527cf35..726a11b0b 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -201,6 +201,26 @@ _TESTS_TO_SKIP = [ # precision to preserve value. Not available on older compilers where we just use absl::Format. # We should probably update the spec to allow different formats that parse to the same value. "conversions/string/double_hard", + + # Recent changes + "proto2/set_null/repeated_field_timestamp_null_pruned", + "proto2/set_null/repeated_field_duration_null_pruned", + "proto2/set_null/repeated_field_wrapper_null_pruned", + "proto2/set_null/map_timestamp_null_pruned", + "proto2/set_null/map_duration_null_pruned", + "proto2/set_null/map_wrapper_null_pruned", + "proto3/set_null/repeated_field_timestamp_null_pruned", + "proto3/set_null/repeated_field_duration_null_pruned", + "proto3/set_null/repeated_field_wrapper_null_pruned", + "proto3/set_null/map_timestamp_null_pruned", + "proto3/set_null/map_duration_null_pruned", + "proto3/set_null/map_wrapper_null_pruned", + "string_ext/format/default precision for fixed-point clause with int", + "string_ext/format/default precision for fixed-point clause with uint", + "string_ext/format/default precision for scientific notation with int", + "string_ext/format/default precision for scientific notation with uint", + "namespace/namespace_shadowing/basic", + "namespace/namespace_shadowing/comprehension_shadowing_namespaced_selector_disambiguation", ] _TESTS_TO_SKIP_MODERN = _TESTS_TO_SKIP diff --git a/conformance/run.bzl b/conformance/run.bzl index 4fcf325c6..d53fd539c 100644 --- a/conformance/run.bzl +++ b/conformance/run.bzl @@ -70,7 +70,7 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, args.append("--skip_check") else: args.append("--noskip_check") - args.append("--skip_tests={}".format(",".join(_expand_tests_to_skip(skip_tests)))) + args.append("--skip_tests=\"{}\"".format(",".join(_expand_tests_to_skip(skip_tests)))) if dashboard: args.append("--dashboard") return args @@ -80,8 +80,8 @@ def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_ name = _conformance_test_name(name, optimize, recursive), args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, skip_tests, dashboard) + ["$(location " + test + ")" for test in data] + select( { - "@platforms//os:windows": ["--skip_tests={}".format(",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS))], - "//conditions:default": ["--skip_tests={}".format(",".join(skip_tests))], + "@platforms//os:windows": ["--skip_tests=\"{}\"".format(",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS))], + "//conditions:default": ["--skip_tests=\"{}\"".format(",".join(skip_tests))], }, ), data = data, From cb9dc8a2e71e503655b1992bdba3debc7fda12a7 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 8 May 2026 15:45:31 -0700 Subject: [PATCH 007/350] Fix command line argument splitting issue for conformance tests. PiperOrigin-RevId: 912731724 --- conformance/run.bzl | 10 +++++----- conformance/run.cc | 8 ++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/conformance/run.bzl b/conformance/run.bzl index d53fd539c..15850b0aa 100644 --- a/conformance/run.bzl +++ b/conformance/run.bzl @@ -56,7 +56,7 @@ def _conformance_test_name(name, optimize, recursive): ], ) -def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, skip_tests, dashboard): +def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard): args = [] if modern: args.append("--modern") @@ -70,7 +70,6 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, args.append("--skip_check") else: args.append("--noskip_check") - args.append("--skip_tests=\"{}\"".format(",".join(_expand_tests_to_skip(skip_tests)))) if dashboard: args.append("--dashboard") return args @@ -78,10 +77,11 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard): cc_test( name = _conformance_test_name(name, optimize, recursive), - args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, skip_tests, dashboard) + ["$(location " + test + ")" for test in data] + select( + args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard) + ["$(location " + test + ")" for test in data], + env = select( { - "@platforms//os:windows": ["--skip_tests=\"{}\"".format(",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS))], - "//conditions:default": ["--skip_tests=\"{}\"".format(",".join(skip_tests))], + "@platforms//os:windows": {"CEL_SKIP_TESTS": ",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS)}, + "//conditions:default": {"CEL_SKIP_TESTS": ",".join(skip_tests)}, }, ), data = data, diff --git a/conformance/run.cc b/conformance/run.cc index d5a919d76..80164d9a4 100644 --- a/conformance/run.cc +++ b/conformance/run.cc @@ -42,6 +42,7 @@ #include "absl/strings/cord.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" #include "absl/types/span.h" @@ -273,6 +274,13 @@ int main(int argc, char** argv) { { auto service = NewConformanceServiceFromFlags(); auto tests_to_skip = absl::GetFlag(FLAGS_skip_tests); + if (const char* env_skip = std::getenv("CEL_SKIP_TESTS"); + env_skip != nullptr) { + for (absl::string_view test : + absl::StrSplit(env_skip, ',', absl::SkipEmpty())) { + tests_to_skip.push_back(std::string(test)); + } + } for (int argi = 1; argi < argc; argi++) { ABSL_CHECK_OK(RegisterTestsFromFile(service, tests_to_skip, absl::string_view(argv[argi]))); From cf31ddf620b9d809014418e82428863b54190cbb Mon Sep 17 00:00:00 2001 From: Tristan Swadell Date: Mon, 11 May 2026 10:50:27 -0700 Subject: [PATCH 008/350] Introduce `Bind` expression factory helper PiperOrigin-RevId: 913778503 --- common/expr_factory.h | 23 ++++++++++++++ parser/macro_expr_factory_test.cc | 51 +++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/common/expr_factory.h b/common/expr_factory.h index b9769b457..773217ad9 100644 --- a/common/expr_factory.h +++ b/common/expr_factory.h @@ -352,6 +352,29 @@ class ExprFactory { return expr; } + template ::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + Expr NewBind(NextIdFunc next_id, BindVar bind_var, BindExpr bind_expr, + RestExpr rest_expr) { + Expr expr; + expr.set_id(next_id()); + auto& comprehension_expr = expr.mutable_comprehension_expr(); + comprehension_expr.set_iter_var("#unused"); + comprehension_expr.set_iter_range( + NewList(next_id(), std::vector{})); + comprehension_expr.set_accu_var(bind_var); + comprehension_expr.set_accu_init(std::move(bind_expr)); + comprehension_expr.set_loop_condition(NewBoolConst(next_id(), false)); + comprehension_expr.set_loop_step(NewIdent(next_id(), bind_var)); + comprehension_expr.set_result(std::move(rest_expr)); + return expr; + } + private: friend class MacroExprFactory; friend class ParserMacroExprFactory; diff --git a/parser/macro_expr_factory_test.cc b/parser/macro_expr_factory_test.cc index 489538be1..b95cbe16f 100644 --- a/parser/macro_expr_factory_test.cc +++ b/parser/macro_expr_factory_test.cc @@ -15,6 +15,7 @@ #include "parser/macro_expr_factory.h" #include +#include #include #include "absl/strings/string_view.h" @@ -39,6 +40,7 @@ class TestMacroExprFactory final : public MacroExprFactory { return NewUnspecified(NextId()); } + using MacroExprFactory::NewBind; using MacroExprFactory::NewBoolConst; using MacroExprFactory::NewCall; using MacroExprFactory::NewComprehension; @@ -69,6 +71,8 @@ class TestMacroExprFactory final : public MacroExprFactory { namespace { +using ::testing::IsEmpty; + TEST(MacroExprFactory, CopyUnspecified) { TestMacroExprFactory factory; EXPECT_EQ(factory.Copy(factory.NewUnspecified()), factory.NewUnspecified(2)); @@ -147,5 +151,52 @@ TEST(MacroExprFactory, CopyComprehension) { factory.NewIdent(11, "foo"), factory.NewIdent(12, "bar"))); } +TEST(MacroExprFactory, NewBind) { + TestMacroExprFactory factory; + Expr bind_expr = factory.NewIdent(10, "x"); + Expr rest_expr = factory.NewIdent(20, "y"); + + auto next_id = [id = 100]() mutable { return id++; }; + + Expr expr = + factory.NewBind(next_id, "a", std::move(bind_expr), std::move(rest_expr)); + + EXPECT_EQ(expr.id(), 100); + ASSERT_TRUE(expr.has_comprehension_expr()); + + const auto& comp = expr.comprehension_expr(); + EXPECT_EQ(comp.iter_var(), "#unused"); + + ASSERT_TRUE(comp.has_iter_range()); + EXPECT_EQ(comp.iter_range().id(), 101); + EXPECT_EQ(comp.iter_range().kind_case(), ExprKindCase::kListExpr); + EXPECT_THAT(comp.iter_range().list_expr().elements(), IsEmpty()); + + EXPECT_EQ(comp.accu_var(), "a"); + + ASSERT_TRUE(comp.has_accu_init()); + Expr expected_bind_expr; + expected_bind_expr.set_id(10); + expected_bind_expr.mutable_ident_expr().set_name("x"); + EXPECT_EQ(comp.accu_init(), expected_bind_expr); + + ASSERT_TRUE(comp.has_loop_condition()); + EXPECT_EQ(comp.loop_condition().id(), 102); + EXPECT_EQ(comp.loop_condition().kind_case(), ExprKindCase::kConstant); + EXPECT_TRUE(comp.loop_condition().const_expr().has_bool_value()); + EXPECT_FALSE(comp.loop_condition().const_expr().bool_value()); + + ASSERT_TRUE(comp.has_loop_step()); + EXPECT_EQ(comp.loop_step().id(), 103); + EXPECT_EQ(comp.loop_step().kind_case(), ExprKindCase::kIdentExpr); + EXPECT_EQ(comp.loop_step().ident_expr().name(), "a"); + + ASSERT_TRUE(comp.has_result()); + Expr expected_rest_expr; + expected_rest_expr.set_id(20); + expected_rest_expr.mutable_ident_expr().set_name("y"); + EXPECT_EQ(comp.result(), expected_rest_expr); +} + } // namespace } // namespace cel From 2e6e9ff4493bfbe0baf883107f3fb7ce6f675d88 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 11 May 2026 21:06:47 -0700 Subject: [PATCH 009/350] Add support for abbreviations and aliases in container configuration for CEL C++ environment YAML. This allows specifying name, abbreviations, and aliases in a container config instead of just a string. The string syntax is preserved as an alternative PiperOrigin-RevId: 914038623 --- env/BUILD | 1 + env/config.h | 11 +++- env/env.cc | 12 +++- env/env_test.cc | 30 ++++++++++ env/env_yaml.cc | 107 +++++++++++++++++++++++++++++++-- env/env_yaml_test.cc | 139 ++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 289 insertions(+), 11 deletions(-) diff --git a/env/BUILD b/env/BUILD index 55297b190..41ffc1723 100644 --- a/env/BUILD +++ b/env/BUILD @@ -52,6 +52,7 @@ cc_library( ":config", "//checker:type_checker_builder", "//common:constant", + "//common:container", "//common:decl", "//common:type", "//compiler", diff --git a/env/config.h b/env/config.h index 10b23d030..e427832ff 100644 --- a/env/config.h +++ b/env/config.h @@ -34,9 +34,16 @@ class Config { struct ContainerConfig { std::string name; - // TODO(uncreated-issue/87): add support for aliases and abbreviations. + std::vector abbreviations; + struct Alias { + std::string alias; + std::string qualified_name; + }; + std::vector aliases; - bool IsEmpty() const { return name.empty(); } + bool IsEmpty() const { + return name.empty() && abbreviations.empty() && aliases.empty(); + } }; void SetContainerConfig(ContainerConfig container_config) { diff --git a/env/env.cc b/env/env.cc index 5a4198497..42652ce59 100644 --- a/env/env.cc +++ b/env/env.cc @@ -24,6 +24,7 @@ #include "absl/strings/string_view.h" #include "checker/type_checker_builder.h" #include "common/constant.h" +#include "common/container.h" #include "common/decl.h" #include "common/type.h" #include "compiler/compiler.h" @@ -130,7 +131,16 @@ absl::StatusOr> Env::NewCompilerBuilder() { cel::TypeCheckerBuilder& checker_builder = compiler_builder->GetCheckerBuilder(); - checker_builder.set_container(config_.GetContainerConfig().name); + ExpressionContainer container; + CEL_RETURN_IF_ERROR( + container.SetContainer(config_.GetContainerConfig().name)); + for (const auto& abbr : config_.GetContainerConfig().abbreviations) { + CEL_RETURN_IF_ERROR(container.AddAbbreviation(abbr)); + } + for (const auto& alias : config_.GetContainerConfig().aliases) { + CEL_RETURN_IF_ERROR(container.AddAlias(alias.alias, alias.qualified_name)); + } + checker_builder.SetExpressionContainer(std::move(container)); if (!config_.GetStandardLibraryConfig().disable) { CEL_RETURN_IF_ERROR( diff --git a/env/env_test.cc b/env/env_test.cc index 076eb57bc..b599aa569 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -314,6 +314,36 @@ TEST(ContainerConfigTest, ContainerConfig) { EXPECT_THAT(result.GetIssues(), IsEmpty()) << result.FormatError(); } +TEST(ContainerConfigTest, ContainerConfigWithAbbreviations) { + Env env; + env.SetDescriptorPool(internal::GetSharedTestingDescriptorPool()); + Config config; + config.SetContainerConfig( + {.name = "cel.expr.conformance", + .abbreviations = {"cel.expr.conformance.proto2.TestAllTypes"}}); + env.SetConfig(config); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, env.NewCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, compiler->Compile("TestAllTypes{}")); + + EXPECT_THAT(result.GetIssues(), IsEmpty()) << result.FormatError(); +} + +TEST(ContainerConfigTest, ContainerConfigWithAliases) { + Env env; + env.SetDescriptorPool(internal::GetSharedTestingDescriptorPool()); + Config config; + config.SetContainerConfig( + {.name = "cel.expr.conformance", + .aliases = { + {.alias = "MyTestType", + .qualified_name = "cel.expr.conformance.proto2.TestAllTypes"}}}); + env.SetConfig(config); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, env.NewCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, compiler->Compile("MyTestType{}")); + + EXPECT_THAT(result.GetIssues(), IsEmpty()) << result.FormatError(); +} + struct VariableConfigWithValueTestCase { Config::VariableConfig variable_config; std::string validate_type_expr; diff --git a/env/env_yaml.cc b/env/env_yaml.cc index 4ba16ea84..159786598 100644 --- a/env/env_yaml.cc +++ b/env/env_yaml.cc @@ -150,12 +150,72 @@ absl::Status ParseName(Config& config, absl::string_view yaml, absl::Status ParseContainerConfig(Config& config, absl::string_view yaml, const YAML::Node& root) { const YAML::Node container = root["container"]; - if (container.IsDefined()) { - if (!container.IsScalar()) { - return YamlError(yaml, container, "Node 'container' is not a string"); - } + if (!container.IsDefined()) { + return absl::OkStatus(); + } + + if (container.IsScalar()) { config.SetContainerConfig({.name = GetString(yaml, container)}); + return absl::OkStatus(); } + + if (!container.IsMap()) { + return YamlError(yaml, container, + "Node 'container' is neither a string nor a map"); + } + + Config::ContainerConfig container_config; + + const YAML::Node name = container["name"]; + if (name.IsDefined()) { + if (!name.IsScalar()) { + return YamlError(yaml, name, "Node 'name' in container is not a string"); + } + container_config.name = GetString(yaml, name); + } + + const YAML::Node abbreviations = container["abbreviations"]; + if (abbreviations.IsDefined()) { + if (!abbreviations.IsSequence()) { + return YamlError(yaml, abbreviations, + "Node 'abbreviations' is not a sequence"); + } + for (const YAML::Node& abbr : abbreviations) { + if (!abbr.IsScalar()) { + return YamlError(yaml, abbr, "Abbreviation is not a string"); + } + container_config.abbreviations.push_back(GetString(yaml, abbr)); + } + } + + const YAML::Node aliases = container["aliases"]; + if (aliases.IsDefined()) { + if (!aliases.IsSequence()) { + return YamlError(yaml, aliases, "Node 'aliases' is not a sequence"); + } + for (const YAML::Node& alias_node : aliases) { + if (!alias_node.IsMap()) { + return YamlError(yaml, alias_node, "Alias entry is not a map"); + } + const YAML::Node alias_key = alias_node["alias"]; + const YAML::Node qualified_name_key = alias_node["qualified_name"]; + + if (!alias_key.IsDefined() || !alias_key.IsScalar()) { + return YamlError(yaml, alias_node, + "Alias entry missing 'alias' string"); + } + if (!qualified_name_key.IsDefined() || !qualified_name_key.IsScalar()) { + return YamlError(yaml, alias_node, + "Alias entry missing 'qualified_name' string"); + } + + container_config.aliases.push_back( + {.alias = GetString(yaml, alias_key), + .qualified_name = GetString(yaml, qualified_name_key)}); + } + } + + config.SetContainerConfig(std::move(container_config)); return absl::OkStatus(); } @@ -686,7 +746,44 @@ void EmitContainerConfig(const Config& env_config, YAML::Emitter& out) { } out << YAML::Key << "container"; - out << YAML::Value << YAML::DoubleQuoted << container_config.name; + if (container_config.abbreviations.empty() && + container_config.aliases.empty()) { + out << YAML::Value << YAML::DoubleQuoted << container_config.name; + } else { + out << YAML::Value << YAML::BeginMap; + if (!container_config.name.empty()) { + out << YAML::Key << "name" << YAML::Value << YAML::DoubleQuoted + << container_config.name; + } + if (!container_config.abbreviations.empty()) { + std::vector sorted_abbrs = container_config.abbreviations; + absl::c_sort(sorted_abbrs); + out << YAML::Key << "abbreviations" << YAML::Value << YAML::BeginSeq; + for (const auto& abbr : sorted_abbrs) { + out << YAML::Value << YAML::DoubleQuoted << abbr; + } + out << YAML::EndSeq; + } + if (!container_config.aliases.empty()) { + std::vector sorted_aliases = + container_config.aliases; + absl::c_sort(sorted_aliases, [](const Config::ContainerConfig::Alias& a, + const Config::ContainerConfig::Alias& b) { + return a.alias < b.alias; + }); + out << YAML::Key << "aliases" << YAML::Value << YAML::BeginSeq; + for (const auto& alias : sorted_aliases) { + out << YAML::BeginMap; + out << YAML::Key << "alias" << YAML::Value << YAML::DoubleQuoted + << alias.alias; + out << YAML::Key << "qualified_name" << YAML::Value + << YAML::DoubleQuoted << alias.qualified_name; + out << YAML::EndMap; + } + out << YAML::EndSeq; + } + out << YAML::EndMap; + } } void EmitExtensionConfigs(const Config& env_config, YAML::Emitter& out) { diff --git a/env/env_yaml_test.cc b/env/env_yaml_test.cc index 25cc63206..d19c0dbfb 100644 --- a/env/env_yaml_test.cc +++ b/env/env_yaml_test.cc @@ -55,6 +55,31 @@ TEST(EnvYamlTest, ParseContainerConfig) { Field(&Config::ContainerConfig::name, "test.container")); } +TEST(EnvYamlTest, ParseContainerConfig_AlternativeSyntax) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + container: + name: test.container + abbreviations: + - abbr1.Abbr1 + - abbr2.Abbr2 + aliases: + - alias: alias1 + qualified_name: qual.name1 + - alias: alias2 + qualified_name: qual.name2 + )yaml")); + + const auto& container_config = config.GetContainerConfig(); + EXPECT_EQ(container_config.name, "test.container"); + EXPECT_THAT(container_config.abbreviations, + UnorderedElementsAre("abbr1.Abbr1", "abbr2.Abbr2")); + ASSERT_THAT(container_config.aliases, SizeIs(2)); + EXPECT_EQ(container_config.aliases[0].alias, "alias1"); + EXPECT_EQ(container_config.aliases[0].qualified_name, "qual.name1"); + EXPECT_EQ(container_config.aliases[1].alias, "alias2"); + EXPECT_EQ(container_config.aliases[1].qualified_name, "qual.name2"); +} + TEST(EnvYamlTest, ParseExtensionConfigs) { ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( extensions: @@ -550,9 +575,78 @@ INSTANTIATE_TEST_SUITE_P( container: - error: "error" )yaml", - .expected_error = "3:19: Node 'container' is not a string\n" - "| - error: \"error\"\n" - "| ^", + .expected_error = + "3:19: Node 'container' is neither a string nor a map\n" + "| - error: \"error\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + name: [] + )yaml", + .expected_error = "3:25: Node 'name' in container is not a string\n" + "| name: []\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + abbreviations: "abbr" + )yaml", + .expected_error = "3:34: Node 'abbreviations' is not a sequence\n" + "| abbreviations: \"abbr\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + abbreviations: + - [] + )yaml", + .expected_error = "4:21: Abbreviation is not a string\n" + "| - []\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + aliases: "not a sequence" + )yaml", + .expected_error = "3:28: Node 'aliases' is not a sequence\n" + "| aliases: \"not a sequence\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + aliases: + - "not a map" + )yaml", + .expected_error = "4:21: Alias entry is not a map\n" + "| - \"not a map\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + aliases: + - qualified_name: "qual" + )yaml", + .expected_error = "4:21: Alias entry missing 'alias' string\n" + "| - qualified_name: \"qual\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + container: + aliases: + - alias: "my_alias" + )yaml", + .expected_error = "4:21: Alias entry missing" + " 'qualified_name' string\n" + "| - alias: \"my_alias\"\n" + "| ^", }, ParseTestCase{ .yaml = R"yaml( @@ -946,6 +1040,33 @@ std::vector GetExportTestCases() { container: "test.container" )yaml", }, + ExportTestCase{ + .config = []() -> absl::StatusOr { + Config config; + config.SetName("test.env"); + config.SetContainerConfig( + {.name = "test.container", + .abbreviations = {"foo", "bar"}, + .aliases = { + {.alias = "foo", .qualified_name = "test.foo"}, + {.alias = "bar", .qualified_name = "test.bar"}, + }}); + return config; + }(), + .expected_yaml = R"yaml( + name: "test.env" + container: + name: "test.container" + abbreviations: + - "bar" + - "foo" + aliases: + - alias: "bar" + qualified_name: "test.bar" + - alias: "foo" + qualified_name: "test.foo" + )yaml", + }, ExportTestCase{ .config = []() -> absl::StatusOr { Config config; @@ -1385,6 +1506,18 @@ std::vector GetRoundTripTestCases() { overloads: - id: "string_to_timestamp" )yaml", + R"yaml( + container: + name: "test.container" + abbreviations: + - "abbr1.Abbr1" + - "abbr2.Abbr2" + aliases: + - alias: "alias1" + qualified_name: "qual.name1" + - alias: "alias2" + qualified_name: "qual.name2" + )yaml", R"yaml( extensions: - name: "bindings" From cd9f059a5833c92576e85e3ffb2eaee2fd328e76 Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 12 May 2026 12:32:02 -0700 Subject: [PATCH 010/350] Fix repeated field null pruning for proto2/proto3 PiperOrigin-RevId: 914421409 --- common/values/struct_value_builder.cc | 11 +++++++++++ conformance/BUILD | 6 ------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index 359596267..c342d6478 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -812,6 +812,17 @@ ProtoMessageRepeatedFieldFromValueMutator( const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + // If the value is null and the target repeated field is anything except + // google.protobuf.{Any,ListValue,Struct,Value}, it should be pruned. + if (value.IsNull()) { + const auto well_known_type = field->message_type()->well_known_type(); + if (well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY) { + return absl::nullopt; + } + } auto* element = reflection->AddMessage(message, field, factory); auto result = ProtoMessageFromValueImpl(value, pool, factory, well_known_types, element); diff --git a/conformance/BUILD b/conformance/BUILD index 726a11b0b..abc0d918a 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -203,15 +203,9 @@ _TESTS_TO_SKIP = [ "conversions/string/double_hard", # Recent changes - "proto2/set_null/repeated_field_timestamp_null_pruned", - "proto2/set_null/repeated_field_duration_null_pruned", - "proto2/set_null/repeated_field_wrapper_null_pruned", "proto2/set_null/map_timestamp_null_pruned", "proto2/set_null/map_duration_null_pruned", "proto2/set_null/map_wrapper_null_pruned", - "proto3/set_null/repeated_field_timestamp_null_pruned", - "proto3/set_null/repeated_field_duration_null_pruned", - "proto3/set_null/repeated_field_wrapper_null_pruned", "proto3/set_null/map_timestamp_null_pruned", "proto3/set_null/map_duration_null_pruned", "proto3/set_null/map_wrapper_null_pruned", From 4749cf81003d9264fd87ca8b0640b5189bcc2b9e Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 12 May 2026 16:00:59 -0700 Subject: [PATCH 011/350] Fix scientific notation and fixed point formatting for int and uint PiperOrigin-RevId: 914525320 --- conformance/BUILD | 4 ---- extensions/formatting.cc | 6 ++++++ extensions/formatting_test.cc | 12 ++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/conformance/BUILD b/conformance/BUILD index abc0d918a..4f9232ab6 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -209,10 +209,6 @@ _TESTS_TO_SKIP = [ "proto3/set_null/map_timestamp_null_pruned", "proto3/set_null/map_duration_null_pruned", "proto3/set_null/map_wrapper_null_pruned", - "string_ext/format/default precision for fixed-point clause with int", - "string_ext/format/default precision for fixed-point clause with uint", - "string_ext/format/default precision for scientific notation with int", - "string_ext/format/default precision for scientific notation with uint", "namespace/namespace_shadowing/basic", "namespace/namespace_shadowing/comprehension_shadowing_namespaced_selector_disambiguation", ] diff --git a/extensions/formatting.cc b/extensions/formatting.cc index 935815569..252fdc7bd 100644 --- a/extensions/formatting.cc +++ b/extensions/formatting.cc @@ -419,6 +419,12 @@ absl::StatusOr GetDouble(const Value& value, std::string& scratch) { str)); } } + if (value.kind() == ValueKind::kInt) { + return static_cast(value.GetInt().NativeValue()); + } + if (value.kind() == ValueKind::kUint) { + return static_cast(value.GetUint().NativeValue()); + } if (value.kind() != ValueKind::kDouble) { return absl::InvalidArgumentError( absl::StrCat("expected a double but got a ", value.GetTypeName())); diff --git a/extensions/formatting_test.cc b/extensions/formatting_test.cc index b80fe9bc0..6a7fb300b 100644 --- a/extensions/formatting_test.cc +++ b/extensions/formatting_test.cc @@ -553,6 +553,18 @@ INSTANTIATE_TEST_SUITE_P( .format_args = "2.71828", .expected = "2.718280e+00", }, + { + .name = "FixedPointClauseWithInt", + .format = "%f", + .format_args = "3", + .expected = "3.000000", + }, + { + .name = "ScientificNotationWithUint", + .format = "%e", + .format_args = "uint(3)", + .expected = "3.000000e+00", + }, { .name = "NaNSupportForFixedPoint", .format = "%f", From 352666fba7822dd0d1f54dc00b332cc527aa81b1 Mon Sep 17 00:00:00 2001 From: Justin King Date: Wed, 13 May 2026 13:07:09 -0700 Subject: [PATCH 012/350] Fix map field value null pruning for proto2/proto3 PiperOrigin-RevId: 915015044 --- common/values/struct_value_builder.cc | 23 +++++++++++++++++++ conformance/BUILD | 6 ----- .../structs/proto_message_type_adapter.cc | 16 +++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index c342d6478..446b18421 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -956,6 +956,19 @@ class MessageValueBuilderImpl { if (error_value) { return false; } + if (map_value_field->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + entry_value.IsNull()) { + auto well_known_type = + map_value_field->message_type()->well_known_type(); + if (well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE && + well_known_type != + google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT) { + return true; + } + } google::protobuf::MapValueRef proto_value; extensions::protobuf_internal::InsertOrLookupMapValue( *reflection_, message_, *field, proto_key, &proto_value); @@ -989,6 +1002,16 @@ class MessageValueBuilderImpl { CEL_RETURN_IF_ERROR(list_value->ForEach( [this, field, accessor, &error_value](const Value& element) -> absl::StatusOr { + if (field->message_type() != nullptr && element.IsNull()) { + auto well_known_type = field->message_type()->well_known_type(); + if (well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE && + well_known_type != + google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT) { + return true; + } + } CEL_ASSIGN_OR_RETURN(error_value, (*accessor)(descriptor_pool_, message_factory_, &well_known_types_, reflection_, diff --git a/conformance/BUILD b/conformance/BUILD index 4f9232ab6..ccd2844c9 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -203,12 +203,6 @@ _TESTS_TO_SKIP = [ "conversions/string/double_hard", # Recent changes - "proto2/set_null/map_timestamp_null_pruned", - "proto2/set_null/map_duration_null_pruned", - "proto2/set_null/map_wrapper_null_pruned", - "proto3/set_null/map_timestamp_null_pruned", - "proto3/set_null/map_duration_null_pruned", - "proto3/set_null/map_wrapper_null_pruned", "namespace/namespace_shadowing/basic", "namespace/namespace_shadowing/comprehension_shadowing_namespaced_selector_disambiguation", ] diff --git a/eval/public/structs/proto_message_type_adapter.cc b/eval/public/structs/proto_message_type_adapter.cc index a351890c2..6a3417ba3 100644 --- a/eval/public/structs/proto_message_type_adapter.cc +++ b/eval/public/structs/proto_message_type_adapter.cc @@ -582,6 +582,19 @@ absl::Status ProtoMessageTypeAdapter::SetField( ValidateSetFieldOp(value_field_descriptor != nullptr, field->name(), "failed to find value field descriptor")); + bool prune_when_null = false; + if (value_field_descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + auto well_known_type = + value_field_descriptor->message_type()->well_known_type(); + if (well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && + well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT) { + prune_when_null = true; + } + } + CEL_ASSIGN_OR_RETURN(const CelList* key_list, cel_map->ListKeys(arena)); for (int i = 0; i < key_list->size(); i++) { CelValue key = (*key_list).Get(arena, i); @@ -589,6 +602,9 @@ absl::Status ProtoMessageTypeAdapter::SetField( auto value = (*cel_map).Get(arena, key); CEL_RETURN_IF_ERROR(ValidateSetFieldOp(value.has_value(), field->name(), "error serializing CelMap")); + if (prune_when_null && value->IsNull()) { + continue; + } Message* entry_msg = message->GetReflection()->AddMessage(message, field); CEL_RETURN_IF_ERROR(internal::SetValueToSingleField( key, key_field_descriptor, entry_msg, arena)); From 037e0bb42339376640024de353451e372bb47820 Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Wed, 13 May 2026 13:33:46 -0700 Subject: [PATCH 013/350] Adding a TypeSpec to Type resolver. PiperOrigin-RevId: 915029748 --- common/BUILD | 32 ++++ common/type_spec_resolver.cc | 182 +++++++++++++++++++++ common/type_spec_resolver.h | 37 +++++ common/type_spec_resolver_test.cc | 257 ++++++++++++++++++++++++++++++ 4 files changed, 508 insertions(+) create mode 100644 common/type_spec_resolver.cc create mode 100644 common/type_spec_resolver.h create mode 100644 common/type_spec_resolver_test.cc diff --git a/common/BUILD b/common/BUILD index 0ead8b15a..ffc4ae1e9 100644 --- a/common/BUILD +++ b/common/BUILD @@ -46,6 +46,38 @@ cc_test( ], ) +cc_library( + name = "type_spec_resolver", + srcs = ["type_spec_resolver.cc"], + hdrs = ["type_spec_resolver.h"], + deps = [ + ":ast", + ":type", + "//internal:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "type_spec_resolver_test", + srcs = ["type_spec_resolver_test.cc"], + deps = [ + ":ast", + ":type", + ":type_kind", + ":type_spec_resolver", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "expr", srcs = ["expr.cc"], diff --git a/common/type_spec_resolver.cc b/common/type_spec_resolver.cc new file mode 100644 index 000000000..97451f390 --- /dev/null +++ b/common/type_spec_resolver.cc @@ -0,0 +1,182 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/type_spec_resolver.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "common/type.h" +#include "internal/status_macros.h" + +namespace cel { + +absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, + google::protobuf::Arena* arena, + const google::protobuf::DescriptorPool& pool) { + if (type_spec.has_null()) return Type(NullType{}); + if (type_spec.has_dyn()) return Type(DynType{}); + + if (type_spec.has_primitive()) { + switch (type_spec.primitive()) { + case PrimitiveType::kBool: + return Type(BoolType{}); + case PrimitiveType::kInt64: + return Type(IntType{}); + case PrimitiveType::kUint64: + return Type(UintType{}); + case PrimitiveType::kDouble: + return Type(DoubleType{}); + case PrimitiveType::kString: + return Type(StringType{}); + case PrimitiveType::kBytes: + return Type(BytesType{}); + default: + return absl::InvalidArgumentError("Unsupported primitive type"); + } + } + + if (type_spec.has_well_known()) { + switch (type_spec.well_known()) { + case WellKnownTypeSpec::kAny: + return Type(AnyType{}); + case WellKnownTypeSpec::kTimestamp: + return Type(TimestampType{}); + case WellKnownTypeSpec::kDuration: + return Type(DurationType{}); + default: + return absl::InvalidArgumentError("Unsupported well-known type"); + } + } + + if (type_spec.has_wrapper()) { + switch (type_spec.wrapper()) { + case PrimitiveType::kBool: + return Type(BoolWrapperType{}); + case PrimitiveType::kInt64: + return Type(IntWrapperType{}); + case PrimitiveType::kUint64: + return Type(UintWrapperType{}); + case PrimitiveType::kDouble: + return Type(DoubleWrapperType{}); + case PrimitiveType::kString: + return Type(StringWrapperType{}); + case PrimitiveType::kBytes: + return Type(BytesWrapperType{}); + default: + return absl::InvalidArgumentError("Unsupported wrapper type"); + } + } + + if (type_spec.has_list_type()) { + CEL_ASSIGN_OR_RETURN( + auto elem_type, + ConvertTypeSpecToType(type_spec.list_type().elem_type(), arena, pool)); + return Type(ListType(arena, elem_type)); + } + + if (type_spec.has_map_type()) { + CEL_ASSIGN_OR_RETURN( + auto key_type, + ConvertTypeSpecToType(type_spec.map_type().key_type(), arena, pool)); + CEL_ASSIGN_OR_RETURN( + auto value_type, + ConvertTypeSpecToType(type_spec.map_type().value_type(), arena, pool)); + return Type(MapType(arena, key_type, value_type)); + } + + if (type_spec.has_function()) { + const auto& func_spec = type_spec.function(); + CEL_ASSIGN_OR_RETURN( + auto result_type, + ConvertTypeSpecToType(func_spec.result_type(), arena, pool)); + std::vector arg_types; + for (const auto& arg_spec : func_spec.arg_types()) { + CEL_ASSIGN_OR_RETURN(auto arg_type, + ConvertTypeSpecToType(arg_spec, arena, pool)); + arg_types.push_back(std::move(arg_type)); + } + return Type(FunctionType(arena, result_type, arg_types)); + } + + if (type_spec.has_type_param()) { + const std::string& name = type_spec.type_param().type(); + auto* allocated_name = google::protobuf::Arena::Create(arena, name); + return Type(TypeParamType(absl::string_view(*allocated_name))); + } + + if (type_spec.has_message_type()) { + const std::string& name = type_spec.message_type().type(); + const google::protobuf::Descriptor* descriptor = pool.FindMessageTypeByName(name); + if (descriptor == nullptr) { + return absl::InvalidArgumentError(absl::StrCat( + "Message type '", name, "' not found in descriptor pool")); + } + return Type::Message(descriptor); + } + + if (type_spec.has_abstract_type()) { + const std::string& name = type_spec.abstract_type().name(); + + // Check if it's a message type in the pool + const google::protobuf::Descriptor* descriptor = pool.FindMessageTypeByName(name); + if (descriptor != nullptr) { + if (!type_spec.abstract_type().parameter_types().empty()) { + return absl::InvalidArgumentError(absl::StrCat( + "Message type '", name, "' cannot have type parameters")); + } + return Type::Message(descriptor); + } + + // Check if it's an enum type in the pool + const google::protobuf::EnumDescriptor* enum_descriptor = + pool.FindEnumTypeByName(name); + if (enum_descriptor != nullptr) { + if (!type_spec.abstract_type().parameter_types().empty()) { + return absl::InvalidArgumentError( + absl::StrCat("Enum type '", name, "' cannot have type parameters")); + } + return Type::Enum(enum_descriptor); + } + + // Otherwise fallback to OpaqueType + std::vector params; + for (const auto& param_spec : type_spec.abstract_type().parameter_types()) { + CEL_ASSIGN_OR_RETURN(auto param, + ConvertTypeSpecToType(param_spec, arena, pool)); + params.push_back(std::move(param)); + } + auto* allocated_name = google::protobuf::Arena::Create(arena, name); + return Type(OpaqueType(arena, absl::string_view(*allocated_name), params)); + } + + if (type_spec.has_type()) { + CEL_ASSIGN_OR_RETURN(auto contained_type, + ConvertTypeSpecToType(type_spec.type(), arena, pool)); + return Type(TypeType(arena, contained_type)); + } + + if (type_spec.has_error()) { + return Type(ErrorType{}); + } + + return absl::InvalidArgumentError("Unknown TypeSpec kind"); +} + +} // namespace cel diff --git a/common/type_spec_resolver.h b/common/type_spec_resolver.h new file mode 100644 index 000000000..44e1e088f --- /dev/null +++ b/common/type_spec_resolver.h @@ -0,0 +1,37 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_COMMON_TYPE_SPEC_RESOLVER_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_TYPE_SPEC_RESOLVER_H_ + +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/type.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" + +namespace cel { + +// Resolves a `cel::TypeSpec` to a `cel::Type`. +// +// TypeSpec only specifies a type while Type provides support for inspecting +// properties of the type when used in CEL. Returns a status with code +// `InvalidArgument` if the input cannot be resolved to a type. +absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, + google::protobuf::Arena* arena, + const google::protobuf::DescriptorPool& pool); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_COMMON_TYPE_SPEC_RESOLVER_H_ diff --git a/common/type_spec_resolver_test.cc b/common/type_spec_resolver_test.cc new file mode 100644 index 000000000..c7fbb2cf8 --- /dev/null +++ b/common/type_spec_resolver_test.cc @@ -0,0 +1,257 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/type_spec_resolver.h" + +#include +#include +#include +#include +#include + +#include "absl/base/no_destructor.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "common/type.h" +#include "common/type_kind.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "google/protobuf/arena.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::cel::internal::GetTestingDescriptorPool; +using ::testing::HasSubstr; +using ::testing::TestWithParam; +using ::testing::Values; + +google::protobuf::Arena* GetTestArena() { + static absl::NoDestructor arena; + return &*arena; +} + +TEST(TypeSpecResolverTest, NullTypeSpec) { + TypeSpec spec(NullTypeSpec{}); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsNull()); +} + +TEST(TypeSpecResolverTest, DynTypeSpec) { + TypeSpec spec(DynTypeSpec{}); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsDyn()); +} + +using ConversionTest = testing::TestWithParam>; + +TEST_P(ConversionTest, TestTypeSpecConversion) { + ASSERT_OK_AND_ASSIGN( + auto t, ConvertTypeSpecToType(std::get<0>(GetParam()), GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_EQ(t.kind(), std::get<1>(GetParam())); +} + +INSTANTIATE_TEST_SUITE_P( + TypeSpecResolverTest, ConversionTest, + testing::Values( + std::make_tuple(TypeSpec(PrimitiveType::kBool), TypeKind::kBool), + std::make_tuple(TypeSpec(PrimitiveType::kInt64), TypeKind::kInt), + std::make_tuple(TypeSpec(PrimitiveType::kUint64), TypeKind::kUint), + std::make_tuple(TypeSpec(PrimitiveType::kDouble), TypeKind::kDouble), + std::make_tuple(TypeSpec(PrimitiveType::kString), TypeKind::kString), + std::make_tuple(TypeSpec(PrimitiveType::kBytes), TypeKind::kBytes), + std::make_tuple(TypeSpec(WellKnownTypeSpec::kAny), TypeKind::kAny), + std::make_tuple(TypeSpec(WellKnownTypeSpec::kTimestamp), + TypeKind::kTimestamp), + std::make_tuple(TypeSpec(WellKnownTypeSpec::kDuration), + TypeKind::kDuration), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)), + TypeKind::kBoolWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)), + TypeKind::kIntWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)), + TypeKind::kUintWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)), + TypeKind::kDoubleWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)), + TypeKind::kStringWrapper), + std::make_tuple(TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)), + TypeKind::kBytesWrapper))); + +TEST(TypeSpecResolverTest, ListTypeConversion) { + auto elem = std::make_unique(PrimitiveType::kInt64); + TypeSpec spec(ListTypeSpec(std::move(elem))); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsList()); + EXPECT_TRUE(t->GetList().element().IsInt()); +} + +TEST(TypeSpecResolverTest, MapTypeConversion) { + auto key = std::make_unique(PrimitiveType::kString); + auto val = std::make_unique(PrimitiveType::kBytes); + TypeSpec spec(MapTypeSpec(std::move(key), std::move(val))); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsMap()); + EXPECT_TRUE(t->GetMap().key().IsString()); + EXPECT_TRUE(t->GetMap().value().IsBytes()); +} + +TEST(TypeSpecResolverTest, FunctionTypeConversion) { + auto result = std::make_unique(PrimitiveType::kBool); + std::vector args; + args.push_back(TypeSpec(PrimitiveType::kString)); + TypeSpec spec(FunctionTypeSpec(std::move(result), std::move(args))); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsFunction()); + EXPECT_EQ(t->GetFunction().args().size(), 1); + EXPECT_TRUE(t->GetFunction().result().IsBool()); +} + +TEST(TypeSpecResolverTest, TypeParamConversion) { + TypeSpec spec(ParamTypeSpec("T")); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsTypeParam()); + EXPECT_EQ(t->GetTypeParam().name(), "T"); +} + +TEST(TypeSpecResolverTest, MessageTypeConversion) { + TypeSpec spec( + AbstractType("cel.expr.conformance.proto3.TestAllTypes", /*params=*/{})); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsMessage()); + EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(TypeSpecResolverTest, MessageTypeWithParamsError) { + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec spec(AbstractType("cel.expr.conformance.proto3.TestAllTypes", + std::move(params))); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(TypeSpecResolverTest, UnresolvedAbstractTypeFallbackToOpaque) { + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec spec(AbstractType("my.custom.OpaqueType", std::move(params))); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsOpaque()); + EXPECT_EQ(t->name(), "my.custom.OpaqueType"); + EXPECT_EQ(t->GetParameters().size(), 1); + EXPECT_TRUE(t->GetParameters()[0].IsInt()); +} + +TEST(TypeSpecResolverTest, OptionalType) { + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec spec(AbstractType("optional_type", std::move(params))); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsOpaque()); + EXPECT_EQ(t->name(), "optional_type"); + EXPECT_EQ(t->GetParameters().size(), 1); + EXPECT_TRUE(t->GetParameters()[0].IsInt()); + EXPECT_TRUE(t->IsOptional()); +} + +TEST(TypeSpecResolverTest, TypeTypeConversion) { + auto nested = std::make_unique(PrimitiveType::kInt64); + TypeSpec spec(std::move(nested)); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsType()); + EXPECT_TRUE(t->GetType().GetType().IsInt()); +} + +TEST(TypeSpecResolverTest, ErrorTypeConversion) { + TypeSpec spec(ErrorTypeSpec::kValue); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsError()); +} + +TEST(TypeSpecResolverTest, MessageTypeSpecConversion) { + TypeSpec spec(MessageTypeSpec("cel.expr.conformance.proto3.TestAllTypes")); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsMessage()); + EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(TypeSpecResolverTest, MessageTypeSpecNotFoundError) { + TypeSpec spec(MessageTypeSpec("cel.expr.conformance.proto3.NonExistentType")); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("not found in descriptor pool"))); +} + +TEST(TypeSpecResolverTest, EnumTypeConversion) { + TypeSpec spec(AbstractType( + "cel.expr.conformance.proto3.TestAllTypes.NestedEnum", /*params=*/{})); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t, IsOk()); + EXPECT_TRUE(t->IsEnum()); + EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"); +} + +TEST(TypeSpecResolverTest, EnumTypeWithParamsError) { + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec spec( + AbstractType("cel.expr.conformance.proto3.TestAllTypes.NestedEnum", + std::move(params))); + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(TypeSpecResolverTest, UnknownTypeSpecKindError) { + TypeSpec spec; + auto t = + ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unknown TypeSpec kind"))); +} + +} // namespace +} // namespace cel From ad18948079b2d3d8b9e62a202889076f872992e7 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 13 May 2026 22:06:31 -0700 Subject: [PATCH 014/350] No public description PiperOrigin-RevId: 915223448 --- eval/public/ast_rewrite.cc | 2 +- eval/public/ast_traverse.cc | 2 +- eval/public/cel_attribute.cc | 4 ++-- eval/public/equality_function_registrar_test.cc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eval/public/ast_rewrite.cc b/eval/public/ast_rewrite.cc index 3c210e607..87c667eb5 100644 --- a/eval/public/ast_rewrite.cc +++ b/eval/public/ast_rewrite.cc @@ -68,7 +68,7 @@ struct ExprRecord { }; using StackRecordKind = - absl::variant; + std::variant; struct StackRecord { public: diff --git a/eval/public/ast_traverse.cc b/eval/public/ast_traverse.cc index a86923c67..c18b806b9 100644 --- a/eval/public/ast_traverse.cc +++ b/eval/public/ast_traverse.cc @@ -67,7 +67,7 @@ struct ExprRecord { }; using StackRecordKind = - absl::variant; + std::variant; struct StackRecord { public: diff --git a/eval/public/cel_attribute.cc b/eval/public/cel_attribute.cc index 015289bed..70525a04d 100644 --- a/eval/public/cel_attribute.cc +++ b/eval/public/cel_attribute.cc @@ -76,8 +76,8 @@ CelAttributeQualifier CreateCelAttributeQualifier(const CelValue& value) { CelAttributePattern CreateCelAttributePattern( absl::string_view variable, - std::initializer_list> + std::initializer_list> path_spec) { std::vector path; path.reserve(path_spec.size()); diff --git a/eval/public/equality_function_registrar_test.cc b/eval/public/equality_function_registrar_test.cc index 772ddfeba..577c4be22 100644 --- a/eval/public/equality_function_registrar_test.cc +++ b/eval/public/equality_function_registrar_test.cc @@ -86,7 +86,7 @@ MATCHER_P2(DefinesHomogenousOverload, name, argument_type, struct EqualityTestCase { enum class ErrorKind { kMissingOverload, kMissingIdentifier }; absl::string_view expr; - absl::variant result; + std::variant result; CelValue lhs = CelValue::CreateNull(); CelValue rhs = CelValue::CreateNull(); }; From 1f5a7e62900ae2ad1021228df04f2a950744c001 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 13 May 2026 22:10:11 -0700 Subject: [PATCH 015/350] No public description PiperOrigin-RevId: 915224564 --- common/ast/constant_proto.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/ast/constant_proto.cc b/common/ast/constant_proto.cc index c0fe1c9f6..1982c05b4 100644 --- a/common/ast/constant_proto.cc +++ b/common/ast/constant_proto.cc @@ -35,7 +35,7 @@ using ConstantProto = cel::expr::Constant; absl::Status ConstantToProto(const Constant& constant, ConstantProto* absl_nonnull proto) { return absl::visit(absl::Overload( - [proto](absl::monostate) -> absl::Status { + [proto](std::monostate) -> absl::Status { proto->clear_constant_kind(); return absl::OkStatus(); }, From 6d311f704ade7aea062dd1091dfe3e683938fc78 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 13 May 2026 22:12:46 -0700 Subject: [PATCH 016/350] No public description PiperOrigin-RevId: 915225296 --- internal/json.cc | 2 +- internal/message_equality.cc | 8 ++++---- internal/well_known_types.cc | 2 +- internal/well_known_types_test.cc | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/json.cc b/internal/json.cc index 630ceb267..cdd4c1a5d 100644 --- a/internal/json.cc +++ b/internal/json.cc @@ -1417,7 +1417,7 @@ class JsonMapIterator final { } private: - absl::variant variant_; + std::variant variant_; }; class JsonAccessor { diff --git a/internal/message_equality.cc b/internal/message_equality.cc index 945cca8df..33ef78089 100644 --- a/internal/message_equality.cc +++ b/internal/message_equality.cc @@ -86,10 +86,10 @@ class EquatableMessage final }; using EquatableValue = - absl::variant; + std::variant; struct NullValueEqualer { bool operator()(std::nullptr_t, std::nullptr_t) const { return true; } diff --git a/internal/well_known_types.cc b/internal/well_known_types.cc index dee029534..02e50c3e3 100644 --- a/internal/well_known_types.cc +++ b/internal/well_known_types.cc @@ -2174,7 +2174,7 @@ absl::StatusOr AdaptFromMessage( if (adapted) { return adapted; } - return absl::monostate{}; + return std::monostate{}; } } diff --git a/internal/well_known_types_test.cc b/internal/well_known_types_test.cc index 0d2c9fe33..afc8ce396 100644 --- a/internal/well_known_types_test.cc +++ b/internal/well_known_types_test.cc @@ -806,7 +806,7 @@ TEST_F(AdaptFromMessageTest, Struct) { TEST_F(AdaptFromMessageTest, TestAllTypesProto3) { auto message = DynamicParseTextProto(R"pb()pb"); EXPECT_THAT(AdaptFromMessage(*message), - IsOkAndHolds(VariantWith(absl::monostate()))); + IsOkAndHolds(VariantWith(std::monostate()))); } TEST_F(AdaptFromMessageTest, Any_BoolValue) { From fb51dcdfd1082e67d209c1ba0c84e58b577c378a Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 00:13:04 -0700 Subject: [PATCH 017/350] No public description PiperOrigin-RevId: 915268033 --- runtime/internal/convert_constant.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/internal/convert_constant.cc b/runtime/internal/convert_constant.cc index a9effd229..33f382858 100644 --- a/runtime/internal/convert_constant.cc +++ b/runtime/internal/convert_constant.cc @@ -33,7 +33,7 @@ using ::cel::Constant; struct ConvertVisitor { Allocator<> allocator; - absl::StatusOr operator()(absl::monostate) { + absl::StatusOr operator()(std::monostate) { return absl::InvalidArgumentError("unspecified constant"); } absl::StatusOr operator()(std::nullptr_t) { return NullValue(); } From 877239571674284da3d22bcc7ccfe2e175643de7 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 00:15:52 -0700 Subject: [PATCH 018/350] No public description PiperOrigin-RevId: 915269088 --- common/values/message_value.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/common/values/message_value.cc b/common/values/message_value.cc index e06206407..66dfd9511 100644 --- a/common/values/message_value.cc +++ b/common/values/message_value.cc @@ -46,7 +46,7 @@ const google::protobuf::Descriptor* absl_nonnull MessageValue::GetDescriptor() c ABSL_CHECK(*this); // Crash OK return absl::visit( absl::Overload( - [](absl::monostate) -> const google::protobuf::Descriptor* absl_nonnull { + [](std::monostate) -> const google::protobuf::Descriptor* absl_nonnull { ABSL_UNREACHABLE(); }, [](const ParsedMessageValue& alternative) @@ -58,7 +58,7 @@ const google::protobuf::Descriptor* absl_nonnull MessageValue::GetDescriptor() c std::string MessageValue::DebugString() const { return absl::visit( - absl::Overload([](absl::monostate) -> std::string { return "INVALID"; }, + absl::Overload([](std::monostate) -> std::string { return "INVALID"; }, [](const ParsedMessageValue& alternative) -> std::string { return alternative.DebugString(); }), @@ -68,7 +68,7 @@ std::string MessageValue::DebugString() const { bool MessageValue::IsZeroValue() const { ABSL_DCHECK(*this); return absl::visit( - absl::Overload([](absl::monostate) -> bool { return true; }, + absl::Overload([](std::monostate) -> bool { return true; }, [](const ParsedMessageValue& alternative) -> bool { return alternative.IsZeroValue(); }), @@ -81,7 +81,7 @@ absl::Status MessageValue::SerializeTo( google::protobuf::io::ZeroCopyOutputStream* absl_nonnull output) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `ConvertToJson` on " "an invalid `MessageValue`"); @@ -99,7 +99,7 @@ absl::Status MessageValue::ConvertToJson( google::protobuf::Message* absl_nonnull json) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `ConvertToJson` on " "an invalid `MessageValue`"); @@ -117,7 +117,7 @@ absl::Status MessageValue::ConvertToJsonObject( google::protobuf::Message* absl_nonnull json) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `ConvertToJsonObject` on " "an invalid `MessageValue`"); @@ -136,7 +136,7 @@ absl::Status MessageValue::Equal( google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `Equal` on " "an invalid `MessageValue`"); @@ -155,7 +155,7 @@ absl::Status MessageValue::GetFieldByName( google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `GetFieldByName` on " "an invalid `MessageValue`"); @@ -175,7 +175,7 @@ absl::Status MessageValue::GetFieldByNumber( google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `GetFieldByNumber` on " "an invalid `MessageValue`"); @@ -192,7 +192,7 @@ absl::StatusOr MessageValue::HasFieldByName( absl::string_view name) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::StatusOr { + [](std::monostate) -> absl::StatusOr { return absl::InternalError( "unexpected attempt to invoke `HasFieldByName` on " "an invalid `MessageValue`"); @@ -206,7 +206,7 @@ absl::StatusOr MessageValue::HasFieldByName( absl::StatusOr MessageValue::HasFieldByNumber(int64_t number) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::StatusOr { + [](std::monostate) -> absl::StatusOr { return absl::InternalError( "unexpected attempt to invoke `HasFieldByNumber` on " "an invalid `MessageValue`"); @@ -224,7 +224,7 @@ absl::Status MessageValue::ForEachField( google::protobuf::Arena* absl_nonnull arena) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `ForEachField` on " "an invalid `MessageValue`"); @@ -244,7 +244,7 @@ absl::Status MessageValue::Qualify( int* absl_nonnull count) const { return absl::visit( absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError( "unexpected attempt to invoke `Qualify` on " "an invalid `MessageValue`"); From ff45a7c2a096ed1d38e6ed4d80a7180be32874b7 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 03:39:55 -0700 Subject: [PATCH 019/350] No public description PiperOrigin-RevId: 915340723 --- common/ast_rewrite.cc | 2 +- common/ast_traverse.cc | 2 +- common/decl_proto.cc | 2 +- common/decl_proto_test.cc | 4 ++-- common/decl_proto_v1alpha1.cc | 2 +- common/type.cc | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/common/ast_rewrite.cc b/common/ast_rewrite.cc index 14582f44f..b61e1fab6 100644 --- a/common/ast_rewrite.cc +++ b/common/ast_rewrite.cc @@ -54,7 +54,7 @@ struct ExprRecord { }; using StackRecordKind = - absl::variant; + std::variant; struct StackRecord { public: diff --git a/common/ast_traverse.cc b/common/ast_traverse.cc index a6ba0d1ba..fb4f9731e 100644 --- a/common/ast_traverse.cc +++ b/common/ast_traverse.cc @@ -53,7 +53,7 @@ struct ExprRecord { }; using StackRecordKind = - absl::variant; + std::variant; struct StackRecord { public: diff --git a/common/decl_proto.cc b/common/decl_proto.cc index 89f7f4453..098c5068c 100644 --- a/common/decl_proto.cc +++ b/common/decl_proto.cc @@ -69,7 +69,7 @@ absl::StatusOr FunctionDeclFromProto( return decl; } -absl::StatusOr> DeclFromProto( +absl::StatusOr> DeclFromProto( const cel::expr::Decl& decl, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::Arena* absl_nonnull arena) { diff --git a/common/decl_proto_test.cc b/common/decl_proto_test.cc index 62215f07f..d72d97e09 100644 --- a/common/decl_proto_test.cc +++ b/common/decl_proto_test.cc @@ -49,7 +49,7 @@ TEST_P(DeclFromProtoTest, FromProtoWorks) { cel::expr::Decl decl_pb; ASSERT_TRUE( google::protobuf::TextFormat::ParseFromString(test_case.proto_decl, &decl_pb)); - absl::StatusOr> decl_or = + absl::StatusOr> decl_or = DeclFromProto(decl_pb, descriptor_pool, &arena); switch (test_case.decl_type) { case DeclType::kVariable: { @@ -79,7 +79,7 @@ TEST_P(DeclFromProtoTest, FromV1Alpha1ProtoWorks) { google::api::expr::v1alpha1::Decl decl_pb; ASSERT_TRUE( google::protobuf::TextFormat::ParseFromString(test_case.proto_decl, &decl_pb)); - absl::StatusOr> decl_or = + absl::StatusOr> decl_or = DeclFromV1Alpha1Proto(decl_pb, descriptor_pool, &arena); switch (test_case.decl_type) { case DeclType::kVariable: { diff --git a/common/decl_proto_v1alpha1.cc b/common/decl_proto_v1alpha1.cc index 2c6cfb6e4..a8d73e5c2 100644 --- a/common/decl_proto_v1alpha1.cc +++ b/common/decl_proto_v1alpha1.cc @@ -52,7 +52,7 @@ absl::StatusOr FunctionDeclFromV1Alpha1Proto( return FunctionDeclFromProto(name, unversioned, descriptor_pool, arena); } -absl::StatusOr> DeclFromV1Alpha1Proto( +absl::StatusOr> DeclFromV1Alpha1Proto( const google::api::expr::v1alpha1::Decl& decl, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::Arena* absl_nonnull arena) { diff --git a/common/type.cc b/common/type.cc index ce8c7a89a..f94e8bc52 100644 --- a/common/type.cc +++ b/common/type.cc @@ -97,7 +97,7 @@ static constexpr std::array kTypeToKindArray = { TypeKind::kUnknown}; static_assert(kTypeToKindArray.size() == - absl::variant_size(), + std::variant_size(), "Kind indexer must match variant declaration for cel::Type."); } // namespace From 366498bd3820ab8382282ca15279753e4789be31 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 03:46:00 -0700 Subject: [PATCH 020/350] No public description PiperOrigin-RevId: 915342975 --- common/types/struct_type.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/types/struct_type.cc b/common/types/struct_type.cc index 4540cec9c..a1be1f786 100644 --- a/common/types/struct_type.cc +++ b/common/types/struct_type.cc @@ -27,7 +27,7 @@ namespace cel { absl::string_view StructType::name() const { ABSL_DCHECK(*this); return absl::visit( - absl::Overload([](absl::monostate) { return absl::string_view(); }, + absl::Overload([](std::monostate) { return absl::string_view(); }, [](const common_internal::BasicStructType& alt) { return alt.name(); }, @@ -39,7 +39,7 @@ TypeParameters StructType::GetParameters() const { ABSL_DCHECK(*this); return absl::visit( absl::Overload( - [](absl::monostate) { return TypeParameters(); }, + [](std::monostate) { return TypeParameters(); }, [](const common_internal::BasicStructType& alt) { return alt.GetParameters(); }, @@ -49,7 +49,7 @@ TypeParameters StructType::GetParameters() const { std::string StructType::DebugString() const { return absl::visit( - absl::Overload([](absl::monostate) { return std::string(); }, + absl::Overload([](std::monostate) { return std::string(); }, [](common_internal::BasicStructType alt) { return alt.DebugString(); }, @@ -72,7 +72,7 @@ MessageType StructType::GetMessage() const { common_internal::TypeVariant StructType::ToTypeVariant() const { return absl::visit( absl::Overload( - [](absl::monostate) { return common_internal::TypeVariant(); }, + [](std::monostate) { return common_internal::TypeVariant(); }, [](common_internal::BasicStructType alt) { return static_cast(alt) ? common_internal::TypeVariant(alt) : common_internal::TypeVariant(); From 33156b1e59b458ff6c24208dbcb66ace3186ab9b Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 04:28:29 -0700 Subject: [PATCH 021/350] No public description PiperOrigin-RevId: 915359111 --- extensions/select_optimization.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 0f09773ae..44da4c48a 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -92,7 +92,7 @@ struct SelectInstruction { // Represents a single qualifier in a traversal path. // TODO(uncreated-issue/51): support variable indexes. using QualifierInstruction = - absl::variant; + std::variant; struct SelectPath { Expr* operand; From a88afaca5106943d9f835cb622be9813b6bdee55 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 05:07:40 -0700 Subject: [PATCH 022/350] No public description PiperOrigin-RevId: 915372200 --- runtime/memory_safety_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/memory_safety_test.cc b/runtime/memory_safety_test.cc index 7e864ecf6..2a09be666 100644 --- a/runtime/memory_safety_test.cc +++ b/runtime/memory_safety_test.cc @@ -73,7 +73,7 @@ struct TestCase { std::string name; std::string expression; absl::flat_hash_map> + std::variant> activation; test::ValueMatcher expected_matcher; bool reference_resolver_enabled = false; From 513af3c2c338c0aabbbef21419018880dc9c23c4 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 22:02:12 -0700 Subject: [PATCH 023/350] No public description PiperOrigin-RevId: 915787513 --- eval/internal/cel_value_equal_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/internal/cel_value_equal_test.cc b/eval/internal/cel_value_equal_test.cc index f52f38916..109a63795 100644 --- a/eval/internal/cel_value_equal_test.cc +++ b/eval/internal/cel_value_equal_test.cc @@ -67,7 +67,7 @@ using ::testing::ValuesIn; struct EqualityTestCase { enum class ErrorKind { kMissingOverload, kMissingIdentifier }; absl::string_view expr; - absl::variant result; + std::variant result; CelValue lhs = CelValue::CreateNull(); CelValue rhs = CelValue::CreateNull(); }; From f62419d04f3a4c12ecf2a802e95d26e33aa2b115 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 22:02:24 -0700 Subject: [PATCH 024/350] No public description PiperOrigin-RevId: 915787600 --- tools/branch_coverage.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/branch_coverage.cc b/tools/branch_coverage.cc index 00ab7cb5a..b5bba3ffe 100644 --- a/tools/branch_coverage.cc +++ b/tools/branch_coverage.cc @@ -71,7 +71,7 @@ struct OtherNode { // Representation for coverage of an AST node. struct CoverageNode { int evaluate_count; - absl::variant kind; + std::variant kind; }; const Type* absl_nullable FindCheckerType(const CheckedExpr& expr, From 7820913cda14e09bbb667c10d65391c1a79fb95d Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 14 May 2026 22:16:10 -0700 Subject: [PATCH 025/350] No public description PiperOrigin-RevId: 915792480 --- common/value.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/common/value.cc b/common/value.cc index 535ddead8..1cd3f54e1 100644 --- a/common/value.cc +++ b/common/value.cc @@ -115,7 +115,7 @@ Type Value::GetRuntimeType() const { namespace { template -struct IsMonostate : std::is_same, absl::monostate> {}; +struct IsMonostate : std::is_same, std::monostate> {}; } // namespace @@ -171,7 +171,7 @@ absl::Status Value::ConvertToJsonArray( google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE); return variant_.Visit(absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError("use of invalid Value"); }, [descriptor_pool, message_factory, json]( @@ -212,7 +212,7 @@ absl::Status Value::ConvertToJsonObject( google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT); return variant_.Visit(absl::Overload( - [](absl::monostate) -> absl::Status { + [](std::monostate) -> absl::Status { return absl::InternalError("use of invalid Value"); }, [descriptor_pool, message_factory, json]( @@ -1363,7 +1363,7 @@ Value Value::FromMessage( return absl::visit( absl::Overload(OwningWellKnownTypesValueVisitor{ /* .arena = */ arena, /* .scratch = */ &scratch}, - [&](absl::monostate) -> Value { + [&](std::monostate) -> Value { auto* cloned = message.New(arena); cloned->CopyFrom(message); return ParsedMessageValue(cloned, arena); @@ -1391,7 +1391,7 @@ Value Value::FromMessage( return absl::visit( absl::Overload(OwningWellKnownTypesValueVisitor{ /* .arena = */ arena, /* .scratch = */ &scratch}, - [&](absl::monostate) -> Value { + [&](std::monostate) -> Value { auto* cloned = message.New(arena); cloned->GetReflection()->Swap(cloned, &message); return ParsedMessageValue(cloned, arena); @@ -1422,7 +1422,7 @@ Value Value::WrapMessage( absl::Overload(BorrowingWellKnownTypesValueVisitor{ /* .message = */ message, /* .arena = */ arena, /* .scratch = */ &scratch}, - [&](absl::monostate) -> Value { + [&](std::monostate) -> Value { if (message->GetArena() != arena) { auto* cloned = message->New(arena); cloned->CopyFrom(*message); @@ -1456,7 +1456,7 @@ Value Value::WrapMessageUnsafe( absl::Overload(BorrowingWellKnownTypesValueVisitor{ /* .message = */ message, /* .arena = */ arena, /* .scratch = */ &scratch}, - [&](absl::monostate) -> Value { + [&](std::monostate) -> Value { if (message->GetArena() != arena) { return UnsafeParsedMessageValue(message); } From da45d34071c8fe9f77fcc17e33e518841d382cdc Mon Sep 17 00:00:00 2001 From: Antoine Pietri Date: Mon, 18 May 2026 08:43:19 -0700 Subject: [PATCH 026/350] Add missing include for `google/rpc/status.proto.h`. This code was relying on the transitive inclusion of third_party/cel/cpp/* to provide the type information for the Status proto. This makes the code brittle and prone to breakages when doing internal header refactors. PiperOrigin-RevId: 917253701 --- conformance/BUILD | 1 + conformance/service.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/conformance/BUILD b/conformance/BUILD index ccd2844c9..0ca90a4bc 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -83,6 +83,7 @@ cc_library( "@com_google_googleapis//google/api/expr/conformance/v1alpha1:conformance_cc_proto", "@com_google_googleapis//google/api/expr/v1alpha1:checked_cc_proto", "@com_google_googleapis//google/rpc:code_cc_proto", + "@com_google_googleapis//google/rpc:status_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/conformance/service.cc b/conformance/service.cc index 463334bb5..7e3eded82 100644 --- a/conformance/service.cc +++ b/conformance/service.cc @@ -30,6 +30,7 @@ #include "google/protobuf/struct.pb.h" #include "google/protobuf/timestamp.pb.h" #include "google/rpc/code.pb.h" +#include "google/rpc/status.pb.h" #include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/status/status.h" From d475cc6726ef85fefed557c8eb0e400119d13e95 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 20 May 2026 20:28:14 -0700 Subject: [PATCH 027/350] No public description PiperOrigin-RevId: 918791547 --- codelab/network_functions.cc | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/codelab/network_functions.cc b/codelab/network_functions.cc index f4f729827..64f199cb3 100644 --- a/codelab/network_functions.cc +++ b/codelab/network_functions.cc @@ -213,8 +213,7 @@ absl::Status NetworkAddressRepEqual( return absl::OkStatus(); } const NetworkAddressRep rep = content.To(); - absl::optional other_rep = - NetworkAddressRep::Unwrap(other); + std::optional other_rep = NetworkAddressRep::Unwrap(other); ABSL_DCHECK(other_rep.has_value()); *result = cel::BoolValue(rep.IsEqualTo(*other_rep)); return absl::OkStatus(); @@ -311,7 +310,7 @@ cel::Value parseAddress( google::protobuf::Arena* absl_nonnull arena) { std::string buf; absl::string_view addr = str.ToStringView(&buf); - absl::optional rep = NetworkAddressRep::Parse(addr); + std::optional rep = NetworkAddressRep::Parse(addr); if (!rep.has_value()) { return cel::ErrorValue(absl::InvalidArgumentError("invalid address")); } @@ -321,7 +320,7 @@ cel::Value parseAddress( cel::Value parseAddressOrZero(const cel::StringValue& str) { std::string buf; absl::string_view addr = str.ToStringView(&buf); - absl::optional rep = NetworkAddressRep::Parse(addr); + std::optional rep = NetworkAddressRep::Parse(addr); static const NetworkAddressRep kZero; if (!rep.has_value()) { return NetworkAddressRep::MakeValue(kZero); @@ -336,8 +335,7 @@ cel::Value parseAddressMatcher( google::protobuf::Arena* absl_nonnull arena) { std::string buf; absl::string_view addr = str.ToStringView(&buf); - absl::optional rep = - NetworkAddressMatcher::Parse(addr); + std::optional rep = NetworkAddressMatcher::Parse(addr); if (!rep.has_value()) { return cel::ErrorValue( absl::InvalidArgumentError("invalid address matcher")); @@ -365,7 +363,7 @@ cel::Value NetworkAddressRep::MakeValue(const NetworkAddressRep& rep) { cel::OpaqueValueContent::From(rep)); } -absl::optional NetworkAddressRep::Unwrap( +std::optional NetworkAddressRep::Unwrap( const cel::Value& value) { auto opaque = value.AsOpaque(); if (!opaque.has_value() || @@ -381,7 +379,7 @@ absl::optional NetworkAddressRep::Unwrap( return opaque->content().To(); } -absl::optional NetworkAddressRep::Parse( +std::optional NetworkAddressRep::Parse( absl::string_view str) { uint32_t ipv4 = 0; char ipv6[16]; @@ -418,7 +416,7 @@ bool NetworkAddressRep::IsLessThan(const NetworkAddressRep& other) const { return false; } -absl::optional NetworkAddressMatcher::Parse( +std::optional NetworkAddressMatcher::Parse( absl::string_view str) { // range style addr-addr int dash_pos = str.find('-'); From b7096df80e0d7b6facc2943326a1c04cde0f1d27 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 20 May 2026 20:28:15 -0700 Subject: [PATCH 028/350] No public description PiperOrigin-RevId: 918791557 --- eval/public/equality_function_registrar_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eval/public/equality_function_registrar_test.cc b/eval/public/equality_function_registrar_test.cc index 577c4be22..a77a92734 100644 --- a/eval/public/equality_function_registrar_test.cc +++ b/eval/public/equality_function_registrar_test.cc @@ -204,7 +204,7 @@ std::string CelValueEqualTestName( } TEST_P(CelValueEqualImplTypesTest, Basic) { - absl::optional result = CelValueEqualImpl(lhs(), rhs()); + std::optional result = CelValueEqualImpl(lhs(), rhs()); if (lhs().IsNull() || rhs().IsNull()) { if (lhs().IsNull() && rhs().IsNull()) { @@ -286,7 +286,7 @@ const std::vector& NumericValuesNotEqualExample() { using NumericInequalityTest = testing::TestWithParam; TEST_P(NumericInequalityTest, NumericValues) { NumericInequalityTestCase test_case = GetParam(); - absl::optional result = CelValueEqualImpl(test_case.a, test_case.b); + std::optional result = CelValueEqualImpl(test_case.a, test_case.b); EXPECT_TRUE(result.has_value()); EXPECT_EQ(*result, false); } @@ -299,7 +299,7 @@ INSTANTIATE_TEST_SUITE_P( }); TEST(CelValueEqualImplTest, LossyNumericEquality) { - absl::optional result = CelValueEqualImpl( + std::optional result = CelValueEqualImpl( CelValue::CreateDouble( static_cast(std::numeric_limits::max()) - 1), CelValue::CreateInt64(std::numeric_limits::max())); From 719f3eed5919bc964b30c7e06a77a3a7eeb64953 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 20 May 2026 21:47:09 -0700 Subject: [PATCH 029/350] No public description PiperOrigin-RevId: 918818689 --- eval/tests/benchmark_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/tests/benchmark_test.cc b/eval/tests/benchmark_test.cc index fc0c39294..f188dc0b7 100644 --- a/eval/tests/benchmark_test.cc +++ b/eval/tests/benchmark_test.cc @@ -317,7 +317,7 @@ BENCHMARK(BM_PolicySymbolic); class RequestMap : public CelMap { public: - absl::optional operator[](CelValue key) const override { + std::optional operator[](CelValue key) const override { if (!key.IsString()) { return {}; } From a552526a3c58346438cec05cbfe3afeb20657ed6 Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Thu, 21 May 2026 12:28:31 -0700 Subject: [PATCH 030/350] Add functions to parse type and function signatures into cel types. PiperOrigin-RevId: 919194450 --- common/internal/BUILD | 8 +- common/internal/signature.cc | 390 +++++++++++++++++++++++- common/internal/signature.h | 21 ++ common/internal/signature_test.cc | 489 ++++++++++++++++++++++++++++-- 4 files changed, 889 insertions(+), 19 deletions(-) diff --git a/common/internal/BUILD b/common/internal/BUILD index 10084b685..48a8dfe8b 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -143,14 +143,17 @@ cc_library( srcs = ["signature.cc"], hdrs = ["signature.h"], deps = [ + "//common:ast", "//common:type", "//common:type_kind", + "//common:type_spec_resolver", "//internal:status_macros", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", + "@com_google_protobuf//:protobuf", ], ) @@ -159,11 +162,14 @@ cc_test( srcs = ["signature_test.cc"], deps = [ ":signature", + "//common:ast", "//common:type", + "//common:type_kind", "//internal:testing", "//internal:testing_descriptor_pool", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_protobuf//:protobuf", ], diff --git a/common/internal/signature.cc b/common/internal/signature.cc index f63049878..5c75225f9 100644 --- a/common/internal/signature.cc +++ b/common/internal/signature.cc @@ -15,20 +15,30 @@ #include "common/internal/signature.h" #include +#include +#include #include #include +#include #include #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/types/optional.h" +#include "common/ast.h" #include "common/type.h" #include "common/type_kind.h" +#include "common/type_spec_resolver.h" #include "internal/status_macros.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" namespace cel::common_internal { +// Signature generator helper functions. namespace { void AppendEscaped(std::string* result, std::string_view str, bool escape_dot) { @@ -58,7 +68,7 @@ absl::Status AppendTypeParameters(std::string* result, const Type& type); // Recursively appends a string representation of the given `type` to `result`. // Type parameters are enclosed in angle brackets and separated by commas. - +// // Grammar: // TypeDesc = NamespaceIdentifier [ "<" TypeList ">" ] ; // NamespaceIdentifier = [ "." ] Identifier { "." Identifier } ; @@ -208,4 +218,382 @@ absl::StatusOr MakeOverloadSignature( return result; } + +// Signature parser helper functions. +namespace { + +std::string StripUnescapedWhitespace(std::string_view str) { + std::string result; + result.reserve(str.size()); + bool escaped = false; + for (char c : str) { + if (escaped) { + result.push_back(c); + escaped = false; + continue; + } + if (c == '\\') { + result.push_back(c); + escaped = true; + continue; + } + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + continue; + } + result.push_back(c); + } + return result; +} + +absl::optional ParseBuiltinOrWrapper(std::string_view name_str) { + if (name_str == "null") return TypeSpec(NullTypeSpec()); + if (name_str == "bool") return TypeSpec(PrimitiveType::kBool); + if (name_str == "int") return TypeSpec(PrimitiveType::kInt64); + if (name_str == "uint") return TypeSpec(PrimitiveType::kUint64); + if (name_str == "double") return TypeSpec(PrimitiveType::kDouble); + if (name_str == "string") return TypeSpec(PrimitiveType::kString); + if (name_str == "bytes") return TypeSpec(PrimitiveType::kBytes); + if (name_str == "any" || name_str == "google.protobuf.Any") + return TypeSpec(WellKnownTypeSpec::kAny); + if (name_str == "timestamp" || name_str == "google.protobuf.Timestamp") + return TypeSpec(WellKnownTypeSpec::kTimestamp); + if (name_str == "duration" || name_str == "google.protobuf.Duration") + return TypeSpec(WellKnownTypeSpec::kDuration); + if (name_str == "dyn" || name_str == "google.protobuf.Value") + return TypeSpec(DynTypeSpec()); + + // Handle standard Protobuf well-known wrapper types to preserve + // backward compatibility for users migrating YAML configuration files. + if (name_str == "bool_wrapper" || name_str == "google.protobuf.BoolValue") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)); + if (name_str == "int_wrapper" || name_str == "google.protobuf.Int64Value" || + name_str == "google.protobuf.Int32Value") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)); + if (name_str == "uint_wrapper" || name_str == "google.protobuf.UInt64Value" || + name_str == "google.protobuf.UInt32Value") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)); + if (name_str == "double_wrapper" || + name_str == "google.protobuf.DoubleValue" || + name_str == "google.protobuf.FloatValue") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)); + if (name_str == "string_wrapper" || name_str == "google.protobuf.StringValue") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)); + if (name_str == "bytes_wrapper" || name_str == "google.protobuf.BytesValue") + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)); + + if (name_str == "google.protobuf.ListValue") { + return TypeSpec(ListTypeSpec(std::make_unique(DynTypeSpec()))); + } + if (name_str == "google.protobuf.Struct") { + return TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kString), + std::make_unique(DynTypeSpec()))); + } + + return absl::nullopt; +} + +std::string Unescape(std::string_view str) { + size_t first_escape = str.find('\\'); + if (first_escape == std::string_view::npos) { + return std::string(str); + } + std::string result; + result.reserve(str.size()); + result.append(str.substr(0, first_escape)); + bool escaped = false; + for (size_t i = first_escape; i < str.size(); ++i) { + char c = str[i]; + if (escaped) { + result.push_back(c); + escaped = false; + } else if (c == '\\') { + escaped = true; + } else { + result.push_back(c); + } + } + if (escaped) { + result.push_back('\\'); + } + return result; +} + +class SignatureScanner { + public: + explicit SignatureScanner(std::string_view input, + std::string_view error_prefix = "Invalid signature") + : input_(input), error_prefix_(error_prefix) {} + + absl::StatusOr FindTopLevelChar(char target, bool find_last = false) { + size_t found_idx = std::string_view::npos; + int nesting = 0; + bool escaped = false; + // Scanning str for delimiter boundaries while ensuring + // brackets are balanced and escape backslashes are bypassed. + for (size_t i = 0; i < input_.size(); ++i) { + char c = input_[i]; + if (escaped) { + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == target && nesting == 0) { + if (find_last || found_idx == std::string_view::npos) { + found_idx = i; + } + } + if (c == '<') { + nesting++; + } else if (c == '>') { + nesting--; + if (nesting < 0) { + return absl::InvalidArgumentError( + absl::StrCat(error_prefix_, ": mismatched brackets")); + } + } + } + if (nesting != 0) { + return absl::InvalidArgumentError( + absl::StrCat(error_prefix_, ": mismatched brackets")); + } + return found_idx; + } + + absl::StatusOr> SplitTopLevel(char delimiter) { + std::vector result; + int nesting = 0; + bool escaped = false; + size_t start = 0; + // Scanning str for delimiter while ensuring brackets are balanced and + // escape backslashes are bypassed. + for (size_t i = 0; i < input_.size(); ++i) { + char c = input_[i]; + if (escaped) { + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == delimiter && nesting == 0) { + result.push_back(input_.substr(start, i - start)); + start = i + 1; + } + if (c == '<') { + nesting++; + } else if (c == '>') { + nesting--; + if (nesting < 0) { + return absl::InvalidArgumentError( + absl::StrCat(error_prefix_, ": mismatched brackets")); + } + } + } + if (nesting != 0) { + return absl::InvalidArgumentError( + absl::StrCat(error_prefix_, ": mismatched brackets")); + } + result.push_back(input_.substr(start)); + return result; + } + + private: + std::string_view input_; + std::string_view error_prefix_; +}; + +absl::StatusOr> SplitTypeList( + std::string_view params) { + return SignatureScanner(params, "Invalid type signature").SplitTopLevel(','); +} + +absl::StatusOr ParseTypeSignature(std::string_view signature) { + if (signature.empty()) { + return absl::InvalidArgumentError("Empty type signature"); + } + + if (signature[0] == '~') { + std::string_view param_name = signature.substr(1); + if (param_name.empty()) { + return absl::InvalidArgumentError( + "Invalid type signature: invalid type parameter name"); + } + CEL_ASSIGN_OR_RETURN(size_t less_idx, + SignatureScanner(param_name) + .FindTopLevelChar('<', /*find_last=*/false)); + CEL_ASSIGN_OR_RETURN(size_t comma_idx, + SignatureScanner(param_name) + .FindTopLevelChar(',', /*find_last=*/false)); + if (less_idx != std::string_view::npos || + comma_idx != std::string_view::npos) { + return absl::InvalidArgumentError( + "Invalid type signature: invalid type parameter name"); + } + return TypeSpec(ParamTypeSpec(Unescape(param_name))); + } + + CEL_ASSIGN_OR_RETURN(size_t less_idx, + SignatureScanner(signature, "Invalid type signature") + .FindTopLevelChar('<', /*find_last=*/false)); + + std::string name_str; + std::vector params; + + if (less_idx != std::string_view::npos) { + // If the signature contains a '<', it must also contain a matching '>'. + if (signature.back() != '>') { + return absl::InvalidArgumentError( + "Invalid type signature: missing closing >"); + } + name_str = Unescape(signature.substr(0, less_idx)); + std::string_view params_str = + signature.substr(less_idx + 1, signature.size() - less_idx - 2); + CEL_ASSIGN_OR_RETURN(auto param_list, SplitTypeList(params_str)); + for (std::string_view param_str : param_list) { + CEL_ASSIGN_OR_RETURN(auto param, ParseTypeSignature(param_str)); + params.push_back(std::move(param)); + } + } else { + name_str = Unescape(signature); + } + + auto read_param_or_dyn = [¶ms](size_t index) { + auto spec = std::make_unique(DynTypeSpec()); + if (params.size() > index) { + *spec = std::move(params[index]); + } + return spec; + }; + + if (!params.empty()) { + if (ParseBuiltinOrWrapper(name_str).has_value()) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid type signature: ", name_str, + " cannot have type parameters")); + } + } else { + if (auto builtin = ParseBuiltinOrWrapper(name_str); builtin.has_value()) { + return *builtin; + } + } + + if (name_str == "type") { + if (params.size() > 1) { + return absl::InvalidArgumentError( + "Invalid type signature: type expects at most 1 parameter"); + } + return TypeSpec(read_param_or_dyn(0)); + } + + if (name_str == "list") { + if (params.size() > 1) { + return absl::InvalidArgumentError( + "Invalid type signature: list expects at most 1 parameter"); + } + return TypeSpec(ListTypeSpec(read_param_or_dyn(0))); + } + + if (name_str == "map") { + if (!params.empty() && params.size() != 2) { + return absl::InvalidArgumentError( + "Invalid type signature: map expects 0 or 2 parameters"); + } + auto key = read_param_or_dyn(0); + auto value = read_param_or_dyn(1); + return TypeSpec(MapTypeSpec(std::move(key), std::move(value))); + } + + if (name_str == "function") { + auto result_type = read_param_or_dyn(0); + std::vector arg_types; + for (size_t i = 1; i < params.size(); ++i) { + arg_types.push_back(std::move(params[i])); + } + return TypeSpec( + FunctionTypeSpec(std::move(result_type), std::move(arg_types))); + } + + if (name_str.empty() || absl::StrContains(name_str, "..")) { + return absl::InvalidArgumentError( + "Invalid type signature: invalid identifier"); + } + + return TypeSpec(AbstractType(name_str, std::move(params))); +} + +} // namespace + +absl::StatusOr ParseFunctionSignature( + std::string_view signature) { + std::string stripped_sig = StripUnescapedWhitespace(signature); + if (stripped_sig.empty()) { + return absl::InvalidArgumentError("Empty function signature"); + } + + CEL_ASSIGN_OR_RETURN( + size_t paren_idx, + SignatureScanner(stripped_sig, "Invalid function signature") + .FindTopLevelChar('(', /*find_last=*/false)); + + if (paren_idx == std::string_view::npos || stripped_sig.back() != ')') { + return absl::InvalidArgumentError("Invalid function signature"); + } + + std::string_view prefix = std::string_view(stripped_sig).substr(0, paren_idx); + std::string_view args_str = + std::string_view(stripped_sig) + .substr(paren_idx + 1, stripped_sig.size() - paren_idx - 2); + + std::vector arg_types; + ParsedFunctionOverload out; + + CEL_ASSIGN_OR_RETURN(size_t dot_idx, + SignatureScanner(prefix, "Invalid function signature") + .FindTopLevelChar('.', /*find_last=*/true)); + + if (dot_idx != std::string_view::npos) { + out.is_member = true; + std::string_view receiver_str = prefix.substr(0, dot_idx); + std::string_view func_str = prefix.substr(dot_idx + 1); + + CEL_ASSIGN_OR_RETURN(auto receiver_param, ParseTypeSignature(receiver_str)); + arg_types.push_back(std::move(receiver_param)); + out.function_name = Unescape(func_str); + } else { + out.is_member = false; + out.function_name = Unescape(prefix); + } + + if (out.function_name.empty()) { + return absl::InvalidArgumentError( + "Invalid function signature: empty function name"); + } + + if (!args_str.empty()) { + CEL_ASSIGN_OR_RETURN(auto arg_list, SplitTypeList(args_str)); + for (std::string_view arg_str : arg_list) { + CEL_ASSIGN_OR_RETURN(auto arg_param, ParseTypeSignature(arg_str)); + arg_types.push_back(std::move(arg_param)); + } + } + + auto result_type = std::make_unique(DynTypeSpec()); + out.signature_type = + TypeSpec(FunctionTypeSpec(std::move(result_type), std::move(arg_types))); + + return out; +} + +absl::StatusOr ParseType(std::string_view signature, google::protobuf::Arena* arena, + const google::protobuf::DescriptorPool& pool) { + std::string stripped_sig = StripUnescapedWhitespace(signature); + CEL_ASSIGN_OR_RETURN(auto type_spec, ParseTypeSignature(stripped_sig)); + return cel::ConvertTypeSpecToType(type_spec, arena, pool); +} + } // namespace cel::common_internal diff --git a/common/internal/signature.h b/common/internal/signature.h index 3f31d8fd1..3fdba4b2e 100644 --- a/common/internal/signature.h +++ b/common/internal/signature.h @@ -20,7 +20,10 @@ #include #include "absl/status/statusor.h" +#include "common/ast.h" #include "common/type.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" namespace cel::common_internal { @@ -56,6 +59,24 @@ absl::StatusOr MakeOverloadSignature( std::string_view function_name, const std::vector& args, bool is_member); +// Parses a string type signature directly into a `cel::Type`. +absl::StatusOr ParseType(std::string_view signature, google::protobuf::Arena* arena, + const google::protobuf::DescriptorPool& pool); + +// A parsed function overload signature with the function name, flag for member +// function, and the function signature type. +struct ParsedFunctionOverload { + std::string function_name; + bool is_member = false; + // The function signature type, configured as a `FunctionTypeSpec`. + TypeSpec signature_type; +}; + +// Parses a string function overload signature directly into a +// `cel::TypeSpec` configured as a `FunctionTypeSpec`. +absl::StatusOr ParseFunctionSignature( + std::string_view signature); + } // namespace cel::common_internal #endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_SIGNATURE_H_ diff --git a/common/internal/signature_test.cc b/common/internal/signature_test.cc index 8e41c70fb..765055f75 100644 --- a/common/internal/signature_test.cc +++ b/common/internal/signature_test.cc @@ -13,13 +13,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include "absl/base/no_destructor.h" #include "absl/status/status.h" +#include "absl/status/status_matchers.h" #include "absl/status/statusor.h" +#include "common/ast.h" #include "common/type.h" +#include "common/type_kind.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "google/protobuf/arena.h" @@ -38,6 +42,101 @@ google::protobuf::Arena* GetTestArena() { return &*arena; } +void VerifyParsedMatchesType(const TypeSpec& parsed, const Type& original) { + switch (original.kind()) { + case TypeKind::kDyn: + EXPECT_TRUE(parsed.has_dyn()); + break; + case TypeKind::kNull: + EXPECT_TRUE(parsed.has_null()); + break; + case TypeKind::kBool: + EXPECT_EQ(parsed.primitive(), PrimitiveType::kBool); + break; + case TypeKind::kInt: + EXPECT_EQ(parsed.primitive(), PrimitiveType::kInt64); + break; + case TypeKind::kUint: + EXPECT_EQ(parsed.primitive(), PrimitiveType::kUint64); + break; + case TypeKind::kDouble: + EXPECT_EQ(parsed.primitive(), PrimitiveType::kDouble); + break; + case TypeKind::kString: + EXPECT_EQ(parsed.primitive(), PrimitiveType::kString); + break; + case TypeKind::kBytes: + EXPECT_EQ(parsed.primitive(), PrimitiveType::kBytes); + break; + case TypeKind::kAny: + EXPECT_EQ(parsed.well_known(), WellKnownTypeSpec::kAny); + break; + case TypeKind::kTimestamp: + EXPECT_EQ(parsed.well_known(), WellKnownTypeSpec::kTimestamp); + break; + case TypeKind::kDuration: + EXPECT_EQ(parsed.well_known(), WellKnownTypeSpec::kDuration); + break; + case TypeKind::kList: + EXPECT_TRUE(parsed.has_list_type()); + if (!original.GetParameters().empty()) { + VerifyParsedMatchesType(parsed.list_type().elem_type(), + original.GetParameters()[0]); + } + break; + case TypeKind::kMap: + EXPECT_TRUE(parsed.has_map_type()); + if (!original.GetParameters().empty()) { + VerifyParsedMatchesType(parsed.map_type().key_type(), + original.GetParameters()[0]); + } + if (original.GetParameters().size() > 1) { + VerifyParsedMatchesType(parsed.map_type().value_type(), + original.GetParameters()[1]); + } + break; + case TypeKind::kBoolWrapper: + case TypeKind::kIntWrapper: + case TypeKind::kUintWrapper: + case TypeKind::kDoubleWrapper: + case TypeKind::kStringWrapper: + case TypeKind::kBytesWrapper: + EXPECT_TRUE(parsed.has_wrapper()); + break; + case TypeKind::kType: + EXPECT_TRUE(parsed.has_type()); + if (!original.GetParameters().empty()) { + VerifyParsedMatchesType(parsed.type(), original.GetParameters()[0]); + } + break; + case TypeKind::kTypeParam: + EXPECT_TRUE(parsed.has_type_param()); + break; + default: + EXPECT_TRUE(parsed.has_abstract_type()); + break; + } +} + +void VerifyTypesEqual(const Type& lhs, const Type& rhs) { + EXPECT_EQ(lhs.kind(), rhs.kind()); + if (lhs.kind() != rhs.kind()) return; + + if (lhs.kind() == TypeKind::kOpaque || lhs.kind() == TypeKind::kStruct || + lhs.kind() == TypeKind::kTypeParam) { + EXPECT_EQ(lhs.name(), rhs.name()); + } + + const auto& lhs_params = lhs.GetParameters(); + const auto& rhs_params = rhs.GetParameters(); + EXPECT_EQ(lhs_params.size(), rhs_params.size()); + if (lhs_params.size() == rhs_params.size()) { + for (size_t i = 0; i < lhs_params.size(); ++i) { + VerifyTypesEqual(lhs_params[i], rhs_params[i]); + } + } +} + struct TypeSignatureTestCase { Type type; std::string expected_signature; @@ -73,10 +172,18 @@ std::vector GetTypeSignatureTestCases() { .type = ListType(GetTestArena(), StringType{}), .expected_signature = "list", }, + { + .type = TypeType(GetTestArena(), IntType{}), + .expected_signature = "type", + }, { .type = ListType(GetTestArena(), TypeParamType("A")), .expected_signature = "list<~A>", }, + { + .type = ListType(GetTestArena(), TypeParamType("A GetTypeSignatureTestCases() { .expected_signature = "map<~B,~C>", }, { - .type = OpaqueType( - GetTestArena(), "bar", - {FunctionType(GetTestArena(), TypeParamType("D"), {})}), - .expected_signature = "bar>", + .type = OpaqueType(GetTestArena(), "bar", + {FunctionType(GetTestArena(), TypeParamType("D"), + {StringType{}, BoolType{}})}), + .expected_signature = "bar>", }, { .type = AnyType{}, @@ -104,10 +211,18 @@ std::vector GetTypeSignatureTestCases() { .type = TimestampType{}, .expected_signature = "timestamp", }, + { + .type = BoolWrapperType{}, + .expected_signature = "bool_wrapper", + }, { .type = IntWrapperType{}, .expected_signature = "int_wrapper", }, + { + .type = UintWrapperType{}, + .expected_signature = "uint_wrapper", + }, { .type = MessageType(GetTestingDescriptorPool()->FindMessageTypeByName( "cel.expr.conformance.proto3.TestAllTypes")), @@ -117,22 +232,32 @@ std::vector GetTypeSignatureTestCases() { .type = ListType(GetTestArena(), TypeParamType(R"(a,b..(d)\e)")), .expected_signature = R"(list<~a\,b\.\\.\(d\)\\e>)", }, - { - .type = UnknownType{}, - .expected_error = - "Type kind: *unknown* is not supported in CEL declarations", - }, - { - .type = ErrorType{}, - .expected_error = - "Type kind: *error* is not supported in CEL declarations", - }, }; } +TEST(TypeSignatureTest, UnsupportedTypes) { + EXPECT_THAT(common_internal::MakeTypeSignature(UnknownType{}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Type kind: *unknown* is not supported"))); + + EXPECT_THAT(common_internal::MakeTypeSignature(ErrorType{}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Type kind: *error* is not supported"))); +} + INSTANTIATE_TEST_SUITE_P(TypeIdTest, TypeSignatureTest, ValuesIn(GetTypeSignatureTestCases())); +TEST_P(TypeSignatureTest, ParseTypeCheck) { + const auto& param = GetParam(); + if (!param.expected_signature.empty() && param.expected_error.empty()) { + auto parsed = ParseType(param.expected_signature, GetTestArena(), + *GetTestingDescriptorPool()); + ASSERT_THAT(parsed, ::absl_testing::IsOk()); + VerifyTypesEqual(*parsed, param.type); + } +} + struct OverloadSignatureTestCase { std::string function_name = "hello"; std::vector args; @@ -202,10 +327,18 @@ std::vector GetOverloadSignatureTestCases() { .args = {TimestampType{}}, .expected_signature = "hello(timestamp)", }, + { + .args = {BoolWrapperType{}}, + .expected_signature = "hello(bool_wrapper)", + }, { .args = {IntWrapperType{}}, .expected_signature = "hello(int_wrapper)", }, + { + .args = {UintWrapperType{}}, + .expected_signature = "hello(uint_wrapper)", + }, { .args = {MessageType( GetTestingDescriptorPool()->FindMessageTypeByName( @@ -213,9 +346,6 @@ std::vector GetOverloadSignatureTestCases() { .expected_signature = "hello(cel.expr.conformance.proto3.TestAllTypes)", }, - {.args = {}, - .is_member = true, - .expected_error = "Member function with no receiver"}, { .args = {StringType{}}, .is_member = true, @@ -231,6 +361,18 @@ std::vector GetOverloadSignatureTestCases() { .is_member = true, .expected_signature = "string.hello(bool,dyn)", }, + { + .function_name = "hello", + .args = {OpaqueType(GetTestArena(), "bar", + {TypeParamType("dummy.type")})}, + .is_member = true, + .expected_signature = R"(bar<~dummy\.type>.hello())", + }, + { + .function_name = "inspect", + .args = {Type(TypeType(GetTestArena(), StringType{}))}, + .expected_signature = "inspect(type)", + }, { .function_name = R"(h.(e),l\o)", .args = {StringType{}, @@ -242,8 +384,321 @@ std::vector GetOverloadSignatureTestCases() { }; } +TEST(OverloadSignatureTest, MemberFunctionNoReceiverError) { + auto signature = common_internal::MakeOverloadSignature("hello", {}, true); + EXPECT_THAT(signature, + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Member function with no receiver"))); +} + INSTANTIATE_TEST_SUITE_P(OverloadIdTest, OverloadSignatureTest, ValuesIn(GetOverloadSignatureTestCases())); +TEST_P(OverloadSignatureTest, ExhaustiveFunctionParseCheck) { + const auto& param = GetParam(); + if (!param.expected_signature.empty()) { + auto parsed = ParseFunctionSignature(param.expected_signature); + ASSERT_THAT(parsed, ::absl_testing::IsOk()); + EXPECT_EQ(parsed->function_name, param.function_name); + EXPECT_EQ(parsed->is_member, param.is_member); + EXPECT_TRUE(parsed->signature_type.has_function()); + const auto& func = parsed->signature_type.function(); + for (size_t i = 0; i < param.args.size(); ++i) { + VerifyParsedMatchesType(func.arg_types()[i], param.args[i]); + } + } +} + +TEST(ParseSignatureTest, ProtoParsing) { + ASSERT_OK_AND_ASSIGN( + auto t1, ParseType("int", GetTestArena(), *GetTestingDescriptorPool())); + EXPECT_TRUE(t1.IsInt()); + + ASSERT_OK_AND_ASSIGN(auto t2, ParseType("list<~A>", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(t2.IsList()); + + ASSERT_OK_AND_ASSIGN(auto t3, ParseType(R"(~abc\)", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(t3.IsTypeParam()); + EXPECT_EQ(t3.GetTypeParam().name(), R"(abc\)"); + + ASSERT_OK_AND_ASSIGN(auto w1, + ParseType("google.protobuf.BoolValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w1.IsBoolWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w2, + ParseType("google.protobuf.Int64Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w2.IsIntWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w3, + ParseType("google.protobuf.Int32Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w3.IsIntWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w4, + ParseType("google.protobuf.UInt64Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w4.IsUintWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w5, + ParseType("google.protobuf.UInt32Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w5.IsUintWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w6, + ParseType("google.protobuf.DoubleValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w6.IsDoubleWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w7, + ParseType("google.protobuf.FloatValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w7.IsDoubleWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w8, + ParseType("google.protobuf.StringValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w8.IsStringWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w9, + ParseType("google.protobuf.BytesValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w9.IsBytesWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w10, ParseType("string_wrapper", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w10.IsStringWrapper()); + + ASSERT_OK_AND_ASSIGN(auto w11, ParseType("bytes_wrapper", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(w11.IsBytesWrapper()); + + ASSERT_OK_AND_ASSIGN(auto gp_any, + ParseType("google.protobuf.Any", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_any.IsAny()); + + ASSERT_OK_AND_ASSIGN(auto gp_timestamp, + ParseType("google.protobuf.Timestamp", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_timestamp.IsTimestamp()); + + ASSERT_OK_AND_ASSIGN(auto gp_duration, + ParseType("google.protobuf.Duration", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_duration.IsDuration()); + + ASSERT_OK_AND_ASSIGN(auto gp_value, + ParseType("google.protobuf.Value", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_value.IsDyn()); + + ASSERT_OK_AND_ASSIGN(auto gp_list_value, + ParseType("google.protobuf.ListValue", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_list_value.IsList()); + + ASSERT_OK_AND_ASSIGN(auto gp_struct, + ParseType("google.protobuf.Struct", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(gp_struct.IsMap()); + + // Legal whitespace handling tests + ASSERT_OK_AND_ASSIGN(auto ws_type1, + ParseType("map < int , string > ", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(ws_type1.IsMap()); + + ASSERT_OK_AND_ASSIGN(auto ws_type2, + ParseType("map\t<\nint\r,\tstring\n>\r", GetTestArena(), + *GetTestingDescriptorPool())); + EXPECT_TRUE(ws_type2.IsMap()); +} + +TEST(ParseSignatureTest, FunctionParsing) { + ASSERT_OK_AND_ASSIGN(auto f1, ParseFunctionSignature("hello(string)")); + EXPECT_TRUE(f1.signature_type.has_function()); + EXPECT_EQ(f1.signature_type.function().arg_types().size(), 1); + + // Legal whitespace handling tests + ASSERT_OK_AND_ASSIGN(auto ws_func1, + ParseFunctionSignature(" hello ( string ) ")); + EXPECT_TRUE(ws_func1.signature_type.has_function()); + EXPECT_EQ(ws_func1.signature_type.function().arg_types().size(), 1); + + ASSERT_OK_AND_ASSIGN(auto ws_func2, + ParseFunctionSignature("\thello\n(\rstring\t)\n\r")); + EXPECT_TRUE(ws_func2.signature_type.has_function()); + EXPECT_EQ(ws_func2.signature_type.function().arg_types().size(), 1); + + ASSERT_OK_AND_ASSIGN(auto f2, ParseFunctionSignature("a.b.c()")); + EXPECT_TRUE(f2.is_member); + EXPECT_EQ(f2.function_name, "c"); +} + +TEST(ParseSignatureTest, ParsingErrors) { + // Mismatched template brackets and parentheses. + EXPECT_THAT( + ParseType("list>", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT( + ParseType("list", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT(ParseType("list><", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT(ParseFunctionSignature("hello(list>)"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT(ParseFunctionSignature("hello(list)"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT(ParseFunctionSignature("foo", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("list expects at most 1 parameter"))); + EXPECT_THAT( + ParseType("map", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("map expects 0 or 2 parameters"))); + EXPECT_THAT(ParseType("map", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("map expects 0 or 2 parameters"))); + + // Enforcing valid function and identifier names. + EXPECT_THAT(ParseFunctionSignature("()"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("empty function name"))); + EXPECT_THAT(ParseFunctionSignature("string.()"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("empty function name"))); + + // Missing closing operators and boundary checks. + EXPECT_THAT( + ParseType("listfoo", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("missing closing >"))); + + EXPECT_THAT(ParseFunctionSignature("hello>(string)"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + EXPECT_THAT( + ParseType("list<", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + + EXPECT_THAT(ParseType("map", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + + EXPECT_THAT(ParseType("map int, string>", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); + + EXPECT_THAT(ParseType("list", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid type signature"))); + + EXPECT_THAT(ParseFunctionSignature("a..b.c()"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid type signature"))); + EXPECT_THAT( + ParseType("list", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Empty type signature"))); + + EXPECT_THAT( + ParseType("~list", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid type signature"))); + + // Checks that builtin types cannot have type parameters. + EXPECT_THAT( + ParseType("int", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(ParseSignatureTest, MessageTypeWithParamsError) { + EXPECT_THAT(ParseType("cel.expr.conformance.proto3.TestAllTypes", + GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(ParseSignatureTest, MissingClosingParenthesisError) { + EXPECT_THAT(ParseFunctionSignature("hello(string"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid function signature"))); + EXPECT_THAT(ParseFunctionSignature("hello)"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid function signature"))); +} + +TEST(ParseSignatureTest, NestedDotsNonMember) { + auto f1 = ParseFunctionSignature( + "my_opaque()"); + ASSERT_THAT(f1, ::absl_testing::IsOk()); + EXPECT_FALSE(f1->is_member); + EXPECT_EQ(f1->function_name, + "my_opaque"); +} + +TEST(ParseSignatureTest, OverlyComplexSignatures) { + auto t1 = ParseType("map>,map>>", + GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t1, ::absl_testing::IsOk()); + EXPECT_TRUE(t1->IsMap()); + + auto t2 = ParseType(R"(~abc\\)", GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t2, ::absl_testing::IsOk()); + EXPECT_TRUE(t2->IsTypeParam()); + EXPECT_EQ(t2->GetTypeParam().name(), R"(abc\)"); + + auto t3 = + ParseType(R"(~abc\\\\)", GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(t3, ::absl_testing::IsOk()); + EXPECT_TRUE(t3->IsTypeParam()); + EXPECT_EQ(t3->GetTypeParam().name(), R"(abc\\)"); + + auto f1 = ParseFunctionSignature( + "bar>,map>.func(string)"); + ASSERT_THAT(f1, ::absl_testing::IsOk()); + EXPECT_TRUE(f1->is_member); + EXPECT_EQ(f1->function_name, "func"); + EXPECT_TRUE(f1->signature_type.has_function()); + EXPECT_EQ(f1->signature_type.function().arg_types().size(), 2); +} + +TEST(ParseSignatureTest, EmptyOrWhitespaceErrors) { + EXPECT_THAT(ParseType("", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Empty type signature"))); + EXPECT_THAT(ParseFunctionSignature(""), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Empty function signature"))); + EXPECT_THAT(ParseType("list>", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Empty type signature"))); +} + } // namespace } // namespace cel::common_internal From 6fd7030b954562c5e5c1c1185066e80cad29fd25 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 21 May 2026 18:17:46 -0700 Subject: [PATCH 031/350] not yet exported PiperOrigin-RevId: 919358445 --- common/expr_factory.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/expr_factory.h b/common/expr_factory.h index 773217ad9..5607d8deb 100644 --- a/common/expr_factory.h +++ b/common/expr_factory.h @@ -32,6 +32,7 @@ namespace cel { class MacroExprFactory; class ParserMacroExprFactory; +class OptimizerExprFactory; class ExprFactory { protected: @@ -378,6 +379,7 @@ class ExprFactory { private: friend class MacroExprFactory; friend class ParserMacroExprFactory; + friend class OptimizerExprFactory; ExprFactory() : accu_var_(kAccumulatorVariableName) {} From ec82288de1338c6d7763fd722d52c3636965ca1e Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 21 May 2026 20:41:46 -0700 Subject: [PATCH 032/350] No public description PiperOrigin-RevId: 919408845 --- .../descriptor_pool_type_introspector.cc | 12 +++--- .../descriptor_pool_type_introspector_test.cc | 4 +- checker/internal/type_check_env.cc | 10 ++--- checker/internal/type_checker_builder_impl.cc | 6 +-- .../type_checker_builder_impl_test.cc | 2 +- checker/internal/type_checker_impl.cc | 20 +++++----- checker/internal/type_inference_context.cc | 8 ++-- .../internal/type_inference_context_test.cc | 40 +++++++++---------- 8 files changed, 51 insertions(+), 51 deletions(-) diff --git a/checker/internal/descriptor_pool_type_introspector.cc b/checker/internal/descriptor_pool_type_introspector.cc index f6001e947..da4f4430b 100644 --- a/checker/internal/descriptor_pool_type_introspector.cc +++ b/checker/internal/descriptor_pool_type_introspector.cc @@ -35,7 +35,7 @@ namespace { // Standard implementation for field lookups. // Avoids building a FieldTable and just checks the DescriptorPool directly. -absl::StatusOr> +absl::StatusOr> FindStructTypeFieldByNameDirectly( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, absl::string_view type, absl::string_view name) { @@ -60,7 +60,7 @@ FindStructTypeFieldByNameDirectly( // Standard implementation for listing fields. // Avoids building a FieldTable and just checks the DescriptorPool directly. absl::StatusOr< - absl::optional>> + std::optional>> ListStructTypeFieldsDirectly( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, absl::string_view type) { @@ -88,7 +88,7 @@ ListStructTypeFieldsDirectly( using Field = DescriptorPoolTypeIntrospector::Field; -absl::StatusOr> +absl::StatusOr> DescriptorPoolTypeIntrospector::FindTypeImpl(absl::string_view name) const { const google::protobuf::Descriptor* absl_nullable descriptor = descriptor_pool_->FindMessageTypeByName(name); @@ -103,7 +103,7 @@ DescriptorPoolTypeIntrospector::FindTypeImpl(absl::string_view name) const { return absl::nullopt; } -absl::StatusOr> +absl::StatusOr> DescriptorPoolTypeIntrospector::FindEnumConstantImpl( absl::string_view type, absl::string_view value) const { const google::protobuf::EnumDescriptor* absl_nullable enum_descriptor = @@ -124,7 +124,7 @@ DescriptorPoolTypeIntrospector::FindEnumConstantImpl( return absl::nullopt; } -absl::StatusOr> +absl::StatusOr> DescriptorPoolTypeIntrospector::FindStructTypeFieldByNameImpl( absl::string_view type, absl::string_view name) const { if (!use_json_name_) { @@ -151,7 +151,7 @@ DescriptorPoolTypeIntrospector::FindStructTypeFieldByNameImpl( } absl::StatusOr< - absl::optional>> + std::optional>> DescriptorPoolTypeIntrospector::ListFieldsForStructTypeImpl( absl::string_view type) const { if (!use_json_name_) { diff --git a/checker/internal/descriptor_pool_type_introspector_test.cc b/checker/internal/descriptor_pool_type_introspector_test.cc index e2fdc9d40..456798744 100644 --- a/checker/internal/descriptor_pool_type_introspector_test.cc +++ b/checker/internal/descriptor_pool_type_introspector_test.cc @@ -117,7 +117,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, internal::GetTestingDescriptorPool()); introspector.set_use_json_name(true); - absl::StatusOr> field = + absl::StatusOr> field = introspector.FindStructTypeFieldByName( "cel.expr.conformance.proto3.TestAllTypes", "singleInt64"); @@ -132,7 +132,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, ListFieldsForStructType) { DescriptorPoolTypeIntrospector introspector( internal::GetTestingDescriptorPool()); absl::StatusOr< - absl::optional>> + std::optional>> fields = introspector.ListFieldsForStructType( "cel.expr.conformance.proto3.TestAllTypes"); ASSERT_THAT(fields, IsOkAndHolds(Optional(SizeIs(260)))); diff --git a/checker/internal/type_check_env.cc b/checker/internal/type_check_env.cc index c080326cb..763d9ba46 100644 --- a/checker/internal/type_check_env.cc +++ b/checker/internal/type_check_env.cc @@ -48,7 +48,7 @@ const FunctionDecl* absl_nullable TypeCheckEnv::LookupFunction( return nullptr; } -absl::StatusOr> TypeCheckEnv::LookupTypeName( +absl::StatusOr> TypeCheckEnv::LookupTypeName( absl::string_view name) const { for (auto iter = type_providers_.begin(); iter != type_providers_.end(); ++iter) { @@ -60,7 +60,7 @@ absl::StatusOr> TypeCheckEnv::LookupTypeName( return absl::nullopt; } -absl::StatusOr> TypeCheckEnv::LookupEnumConstant( +absl::StatusOr> TypeCheckEnv::LookupEnumConstant( absl::string_view type, absl::string_view value) const { for (auto iter = type_providers_.begin(); iter != type_providers_.end(); ++iter) { @@ -77,9 +77,9 @@ absl::StatusOr> TypeCheckEnv::LookupEnumConstant( return absl::nullopt; } -absl::StatusOr> TypeCheckEnv::LookupTypeConstant( +absl::StatusOr> TypeCheckEnv::LookupTypeConstant( google::protobuf::Arena* absl_nonnull arena, absl::string_view name) const { - CEL_ASSIGN_OR_RETURN(absl::optional type, LookupTypeName(name)); + CEL_ASSIGN_OR_RETURN(std::optional type, LookupTypeName(name)); if (type.has_value()) { return MakeVariableDecl(type->name(), TypeType(arena, *type)); } @@ -94,7 +94,7 @@ absl::StatusOr> TypeCheckEnv::LookupTypeConstant( return absl::nullopt; } -absl::StatusOr> TypeCheckEnv::LookupStructField( +absl::StatusOr> TypeCheckEnv::LookupStructField( absl::string_view type_name, absl::string_view field_name) const { // Check the type providers in registration order. // Note: this doesn't allow for shadowing a type with a subset type of the diff --git a/checker/internal/type_checker_builder_impl.cc b/checker/internal/type_checker_builder_impl.cc index 94a05602e..85b581e83 100644 --- a/checker/internal/type_checker_builder_impl.cc +++ b/checker/internal/type_checker_builder_impl.cc @@ -158,8 +158,8 @@ absl::StatusOr MergeFunctionDecls( return merged_decl; } -absl::optional FilterDecl(FunctionDecl decl, - const TypeCheckerSubset& subset) { +std::optional FilterDecl(FunctionDecl decl, + const TypeCheckerSubset& subset) { FunctionDecl filtered; std::string name = decl.release_name(); std::vector overloads = decl.release_overloads(); @@ -283,7 +283,7 @@ absl::Status TypeCheckerBuilderImpl::ApplyConfig( for (FunctionDeclRecord& fn : config.functions) { FunctionDecl decl = std::move(fn.decl); if (subset != nullptr) { - absl::optional filtered = + std::optional filtered = FilterDecl(std::move(decl), *subset); if (!filtered.has_value()) { continue; diff --git a/checker/internal/type_checker_builder_impl_test.cc b/checker/internal/type_checker_builder_impl_test.cc index f7a3dff97..494e7e440 100644 --- a/checker/internal/type_checker_builder_impl_test.cc +++ b/checker/internal/type_checker_builder_impl_test.cc @@ -144,7 +144,7 @@ TEST(ContextDeclsTest, CustomStructNotSupported) { {}); class MyTypeProvider : public cel::TypeIntrospector { public: - absl::StatusOr> FindTypeImpl( + absl::StatusOr> FindTypeImpl( absl::string_view name) const override { if (name == "com.example.MyStruct") { return common_internal::MakeBasicStructType("com.example.MyStruct"); diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 2472d7def..1ce871255 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -379,7 +379,7 @@ class ResolveVisitor : public AstVisitorBase { // Lookup message type by name to support WellKnownType creation. CEL_ASSIGN_OR_RETURN( - absl::optional field_info, + std::optional field_info, env_->LookupStructField(resolved_name, field.name())); if (!field_info.has_value()) { ReportUndefinedField(field.id(), field.name(), resolved_name); @@ -405,8 +405,8 @@ class ResolveVisitor : public AstVisitorBase { return absl::OkStatus(); } - absl::optional CheckFieldType(int64_t expr_id, const Type& operand_type, - absl::string_view field_name); + std::optional CheckFieldType(int64_t expr_id, const Type& operand_type, + absl::string_view field_name); void HandleOptSelect(const Expr& expr); void HandleBlockIndex(const Expr* expr); @@ -919,7 +919,7 @@ void ResolveVisitor::ResolveFunctionOverloads(const Expr& expr, arg_types.push_back(GetDeducedType(&expr.call_expr().args()[i])); } - absl::optional resolution = + std::optional resolution = inference_context_->ResolveOverload(decl, arg_types, is_receiver); if (!resolution.has_value()) { @@ -968,7 +968,7 @@ const VariableDecl* absl_nullable ResolveVisitor::LookupGlobalIdentifier( if (const VariableDecl* decl = env_->LookupVariable(name); decl != nullptr) { return decl; } - absl::StatusOr> constant = + absl::StatusOr> constant = env_->LookupTypeConstant(arena_, name); if (!constant.ok()) { @@ -1079,9 +1079,9 @@ void ResolveVisitor::ResolveQualifiedIdentifier( } } -absl::optional ResolveVisitor::CheckFieldType(int64_t id, - const Type& operand_type, - absl::string_view field) { +std::optional ResolveVisitor::CheckFieldType(int64_t id, + const Type& operand_type, + absl::string_view field) { if (operand_type.kind() == TypeKind::kDyn || operand_type.kind() == TypeKind::kAny) { return DynType(); @@ -1137,7 +1137,7 @@ void ResolveVisitor::ResolveSelectOperation(const Expr& expr, const Expr& operand) { const Type& operand_type = GetDeducedType(&operand); - absl::optional result_type; + std::optional result_type; int64_t id = expr.id(); // Support short-hand optional chaining. if (operand_type.IsOptional()) { @@ -1184,7 +1184,7 @@ void ResolveVisitor::HandleOptSelect(const Expr& expr) { operand_type = operand_type.GetOptional().GetParameter(); } - absl::optional field_type = CheckFieldType( + std::optional field_type = CheckFieldType( expr.id(), operand_type, field->const_expr().string_value()); if (!field_type.has_value()) { types_[&expr] = ErrorType(); diff --git a/checker/internal/type_inference_context.cc b/checker/internal/type_inference_context.cc index 96d985071..5b909d982 100644 --- a/checker/internal/type_inference_context.cc +++ b/checker/internal/type_inference_context.cc @@ -133,7 +133,7 @@ FunctionOverloadInstance InstantiateFunctionOverload( // Converts a wrapper type to its corresponding primitive type. // Returns nullopt if the type is not a wrapper type. -absl::optional WrapperToPrimitive(const Type& t) { +std::optional WrapperToPrimitive(const Type& t) { switch (t.kind()) { case TypeKind::kBoolWrapper: return BoolType(); @@ -286,7 +286,7 @@ bool TypeInferenceContext::IsAssignableInternal( } // Type is as concrete as it can be under current substitutions. - if (absl::optional wrapped_type = WrapperToPrimitive(to_subs); + if (std::optional wrapped_type = WrapperToPrimitive(to_subs); wrapped_type.has_value()) { return from_subs.IsNull() || IsAssignableInternal(*wrapped_type, from_subs, @@ -531,11 +531,11 @@ bool TypeInferenceContext::IsAssignableWithConstraints( return false; } -absl::optional +std::optional TypeInferenceContext::ResolveOverload(const FunctionDecl& decl, absl::Span argument_types, bool is_receiver) { - absl::optional result_type; + std::optional result_type; std::vector matching_overloads; for (const auto& ovl : decl.overloads()) { diff --git a/checker/internal/type_inference_context_test.cc b/checker/internal/type_inference_context_test.cc index d1bf7fa6d..458d08ff1 100644 --- a/checker/internal/type_inference_context_test.cc +++ b/checker/internal/type_inference_context_test.cc @@ -291,7 +291,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadBasic) { MakeOverloadDecl("add_double", DoubleType(), DoubleType(), DoubleType()))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {IntType(), IntType()}, false); ASSERT_TRUE(resolution.has_value()); EXPECT_THAT(resolution->result_type, IsTypeKind(TypeKind::kInt)); @@ -309,7 +309,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadFails) { MakeOverloadDecl("add_double", DoubleType(), DoubleType(), DoubleType()))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {IntType(), DoubleType()}, false); ASSERT_FALSE(resolution.has_value()); } @@ -324,7 +324,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithParamsNoMatch) { "_==_", MakeOverloadDecl("equals", BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {IntType(), DoubleType()}, false); ASSERT_FALSE(resolution.has_value()); } @@ -341,7 +341,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithMixedParamsMatch) { "_==_", MakeOverloadDecl("equals", BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {list_of_a, list_of_a}, false); ASSERT_TRUE(resolution.has_value()) << context.DebugString(); } @@ -359,7 +359,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithMixedParamsMatch2) { "_==_", MakeOverloadDecl("equals", BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {list_of_a, list_of_int}, false); ASSERT_TRUE(resolution.has_value()) << context.DebugString(); EXPECT_THAT(resolution->overloads, ElementsAre(IsOverloadDecl("equals"))); @@ -375,7 +375,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithParamsMatches) { "_==_", MakeOverloadDecl("equals", BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {IntType(), IntType()}, false); ASSERT_TRUE(resolution.has_value()); EXPECT_TRUE(resolution->result_type.IsBool()); @@ -394,7 +394,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithNestedParamsMatch) { Type list_of_a_instance = context.InstantiateTypeParams(list_of_a); - absl::optional resolution = + std::optional resolution = context.ResolveOverload( decl, {list_of_a_instance, ListType(&arena, IntType())}, false); ASSERT_TRUE(resolution.has_value()); @@ -407,7 +407,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithNestedParamsMatch) { EXPECT_THAT(resolution->overloads, ElementsAre(IsOverloadDecl("add_list"))); - absl::optional resolution2 = + std::optional resolution2 = context.ResolveOverload( decl, {ListType(&arena, IntType()), list_of_a_instance}, false); ASSERT_TRUE(resolution2.has_value()); @@ -433,7 +433,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithNestedParamsNoMatch) { Type list_of_a_instance = context.InstantiateTypeParams(list_of_a); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {list_of_a_instance, IntType()}, false); EXPECT_FALSE(resolution.has_value()); } @@ -450,13 +450,13 @@ TEST(TypeInferenceContextTest, InferencesAccumulate) { Type list_of_a_instance = context.InstantiateTypeParams(list_of_a); - absl::optional resolution1 = + std::optional resolution1 = context.ResolveOverload(decl, {list_of_a_instance, list_of_a_instance}, false); ASSERT_TRUE(resolution1.has_value()); EXPECT_TRUE(resolution1->result_type.IsList()); - absl::optional resolution2 = + std::optional resolution2 = context.ResolveOverload( decl, {resolution1->result_type, ListType(&arena, IntType())}, false); ASSERT_TRUE(resolution2.has_value()); @@ -480,7 +480,7 @@ TEST(TypeInferenceContextTest, DebugString) { MakeFunctionDecl("_+_", MakeOverloadDecl("add_list", list_of_a, list_of_a, list_of_a))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {list_of_int, list_of_int}, false); ASSERT_TRUE(resolution.has_value()); EXPECT_TRUE(resolution->result_type.IsList()); @@ -517,7 +517,7 @@ class TypeInferenceContextWrapperTypesTest TEST_P(TypeInferenceContextWrapperTypesTest, ResolvePrimitiveArg) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload(ternary_decl_, {BoolType(), test_case.wrapper_type, test_case.wrapped_primitive_type}, @@ -534,7 +534,7 @@ TEST_P(TypeInferenceContextWrapperTypesTest, ResolvePrimitiveArg) { TEST_P(TypeInferenceContextWrapperTypesTest, ResolveWrapperArg) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload( ternary_decl_, {BoolType(), test_case.wrapper_type, test_case.wrapper_type}, false); @@ -550,7 +550,7 @@ TEST_P(TypeInferenceContextWrapperTypesTest, ResolveWrapperArg) { TEST_P(TypeInferenceContextWrapperTypesTest, ResolveNullArg) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload(ternary_decl_, {BoolType(), test_case.wrapper_type, NullType()}, false); @@ -566,7 +566,7 @@ TEST_P(TypeInferenceContextWrapperTypesTest, ResolveNullArg) { TEST_P(TypeInferenceContextWrapperTypesTest, NullWidens) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload(ternary_decl_, {BoolType(), NullType(), test_case.wrapper_type}, false); @@ -582,7 +582,7 @@ TEST_P(TypeInferenceContextWrapperTypesTest, NullWidens) { TEST_P(TypeInferenceContextWrapperTypesTest, PrimitiveWidens) { const TypeInferenceContextWrapperTypesTestCase& test_case = GetParam(); - absl::optional resolution = + std::optional resolution = context_.ResolveOverload(ternary_decl_, {BoolType(), test_case.wrapped_primitive_type, test_case.wrapper_type}, @@ -622,7 +622,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithUnionTypePromotion) { /*result_type=*/TypeParamType("A"), BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {BoolType(), NullType(), IntWrapperType()}, false); ASSERT_TRUE(resolution.has_value()); @@ -648,7 +648,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithTypeType) { TypeType(&arena, TypeParamType("A")), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(decl, {StringType()}, false); ASSERT_TRUE(resolution.has_value()); @@ -680,7 +680,7 @@ TEST(TypeInferenceContextTest, ResolveOverloadWithInferredTypeType) { BoolType(), TypeParamType("A"), TypeParamType("A")))); - absl::optional resolution = + std::optional resolution = context.ResolveOverload(to_type_decl, {StringType()}, false); ASSERT_TRUE(resolution.has_value()); From 833bb0c8fe93dc9bf5e2971c38254f80d3a42c1f Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 21 May 2026 20:46:26 -0700 Subject: [PATCH 033/350] No public description PiperOrigin-RevId: 919410392 --- testutil/test_macros.cc | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/testutil/test_macros.cc b/testutil/test_macros.cc index 158135762..672439dc5 100644 --- a/testutil/test_macros.cc +++ b/testutil/test_macros.cc @@ -37,9 +37,8 @@ bool IsCelNamespace(const Expr& target) { return target.has_ident_expr() && target.ident_expr().name() == "cel"; } -absl::optional CelBlockMacroExpander(MacroExprFactory& factory, - Expr& target, - absl::Span args) { +std::optional CelBlockMacroExpander(MacroExprFactory& factory, + Expr& target, absl::Span args) { if (!IsCelNamespace(target)) { return absl::nullopt; } @@ -51,9 +50,8 @@ absl::optional CelBlockMacroExpander(MacroExprFactory& factory, return factory.NewCall("cel.@block", args); } -absl::optional CelIndexMacroExpander(MacroExprFactory& factory, - Expr& target, - absl::Span args) { +std::optional CelIndexMacroExpander(MacroExprFactory& factory, + Expr& target, absl::Span args) { if (!IsCelNamespace(target)) { return absl::nullopt; } @@ -70,9 +68,9 @@ absl::optional CelIndexMacroExpander(MacroExprFactory& factory, return factory.NewIdent(absl::StrCat("@index", index)); } -absl::optional CelIterVarMacroExpander(MacroExprFactory& factory, - Expr& target, - absl::Span args) { +std::optional CelIterVarMacroExpander(MacroExprFactory& factory, + Expr& target, + absl::Span args) { if (!IsCelNamespace(target)) { return absl::nullopt; } @@ -94,9 +92,9 @@ absl::optional CelIterVarMacroExpander(MacroExprFactory& factory, unique_arg.const_expr().int_value())); } -absl::optional CelAccuVarMacroExpander(MacroExprFactory& factory, - Expr& target, - absl::Span args) { +std::optional CelAccuVarMacroExpander(MacroExprFactory& factory, + Expr& target, + absl::Span args) { if (!IsCelNamespace(target)) { return absl::nullopt; } From e00189c323d42b0cacafc320aa890eb4d630d394 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sat, 23 May 2026 02:26:10 -0700 Subject: [PATCH 034/350] No public description PiperOrigin-RevId: 920098804 --- runtime/activation_test.cc | 4 ++-- runtime/function_registry.cc | 5 ++--- runtime/function_registry_test.cc | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/runtime/activation_test.cc b/runtime/activation_test.cc index e6a74f027..4303116a3 100644 --- a/runtime/activation_test.cc +++ b/runtime/activation_test.cc @@ -326,7 +326,7 @@ TEST_F(ActivationTest, MoveAssignment) { "val_provided", [](absl::string_view name, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, google::protobuf::Arena* absl_nonnull) - -> absl::StatusOr> { return IntValue(42); })); + -> absl::StatusOr> { return IntValue(42); })); moved_from.SetUnknownPatterns( {AttributePattern("var1", {AttributeQualifierPattern::OfString("field1")}), @@ -377,7 +377,7 @@ TEST_F(ActivationTest, MoveCtor) { "val_provided", [](absl::string_view name, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, google::protobuf::Arena* absl_nonnull) - -> absl::StatusOr> { return IntValue(42); })); + -> absl::StatusOr> { return IntValue(42); })); moved_from.SetUnknownPatterns( {AttributePattern("var1", {AttributeQualifierPattern::OfString("field1")}), diff --git a/runtime/function_registry.cc b/runtime/function_registry.cc index ac1e53eb5..59f267255 100644 --- a/runtime/function_registry.cc +++ b/runtime/function_registry.cc @@ -44,14 +44,13 @@ class ActivationFunctionProviderImpl public: ActivationFunctionProviderImpl() = default; - absl::StatusOr> GetFunction( + absl::StatusOr> GetFunction( const cel::FunctionDescriptor& descriptor, const cel::ActivationInterface& activation) const override { std::vector overloads = activation.FindFunctionOverloads(descriptor.name()); - absl::optional matching_overload = - absl::nullopt; + std::optional matching_overload = absl::nullopt; for (const auto& overload : overloads) { if (overload.descriptor.ShapeMatches(descriptor)) { diff --git a/runtime/function_registry_test.cc b/runtime/function_registry_test.cc index af7f5bc06..53916777a 100644 --- a/runtime/function_registry_test.cc +++ b/runtime/function_registry_test.cc @@ -120,7 +120,7 @@ TEST(FunctionRegistryTest, DefaultLazyProviderNoOverloadFound) { ASSERT_THAT(providers, SizeIs(1)); const FunctionProvider& provider = providers[0].provider; ASSERT_OK_AND_ASSIGN( - absl::optional func, + std::optional func, provider.GetFunction({"LazyFunc", false, {cel::Kind::kInt64}}, activation)); @@ -146,7 +146,7 @@ TEST(FunctionRegistryTest, DefaultLazyProviderReturnsImpl) { ASSERT_THAT(providers, SizeIs(1)); const FunctionProvider& provider = providers[0].provider; ASSERT_OK_AND_ASSIGN( - absl::optional func, + std::optional func, provider.GetFunction( FunctionDescriptor("LazyFunction", false, {Kind::kInt}), activation)); From 267f4de9814c320e61b20cdd8fbe6580f2e57ac3 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sat, 23 May 2026 02:51:01 -0700 Subject: [PATCH 035/350] No public description PiperOrigin-RevId: 920105999 --- extensions/select_optimization.cc | 16 ++++++++-------- extensions/select_optimization_test.cc | 5 +++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 44da4c48a..42cad0f92 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -153,7 +153,7 @@ Expr MakeSelectPathExpr( // Returns a single select operation based on the inferred type of the operand // and the field name. If the operand type doesn't define the field, returns // nullopt. -absl::optional GetSelectInstruction( +std::optional GetSelectInstruction( const StructType& runtime_type, PlannerContext& planner_context, absl::string_view field_name) { auto field_or = planner_context.type_reflector() @@ -407,13 +407,13 @@ class RewriterImpl : public AstRewriterBase { // support message traversal. const TypeSpec checker_type = ast_.GetTypeOrDyn(operand.id()); - absl::optional rt_type = + std::optional rt_type = (checker_type.has_message_type()) ? GetRuntimeType(checker_type.message_type().type()) : absl::nullopt; if (rt_type.has_value() && (*rt_type).Is()) { const StructType& runtime_type = rt_type->GetStruct(); - absl::optional field_or = + std::optional field_or = GetSelectInstruction(runtime_type, planner_context_, field_name); if (field_or.has_value()) { candidates_[&expr] = std::move(field_or).value(); @@ -538,7 +538,7 @@ class RewriterImpl : public AstRewriterBase { return candidates_.find(operand) != candidates_.end(); } - absl::optional GetRuntimeType(absl::string_view type_name) { + std::optional GetRuntimeType(absl::string_view type_name) { return planner_context_.type_reflector().FindType(type_name).value_or( absl::nullopt); } @@ -582,14 +582,14 @@ class OptimizedSelectImpl { AttributeTrail GetAttributeTrail(const AttributeTrail& operand_trail) const; - absl::optional attribute() const { return attribute_; } + std::optional attribute() const { return attribute_; } const std::vector& qualifiers() const { return qualifiers_; } private: - absl::optional attribute_; + std::optional attribute_; std::vector select_path_; std::vector qualifiers_; bool presence_test_; @@ -597,7 +597,7 @@ class OptimizedSelectImpl { }; // Check for unknowns or missing attributes. -absl::StatusOr> CheckForMarkedAttributes( +absl::StatusOr> CheckForMarkedAttributes( ExecutionFrameBase& frame, const AttributeTrail& attribute_trail) { if (attribute_trail.empty()) { return absl::nullopt; @@ -715,7 +715,7 @@ absl::Status StackMachineImpl::Evaluate(ExecutionFrame* frame) const { // select arguments. // TODO(uncreated-issue/51): add support variable qualifiers attribute_trail = GetAttributeTrail(frame); - CEL_ASSIGN_OR_RETURN(absl::optional value, + CEL_ASSIGN_OR_RETURN(std::optional value, CheckForMarkedAttributes(*frame, attribute_trail)); if (value.has_value()) { frame->value_stack().Pop(kStackInputs); diff --git a/extensions/select_optimization_test.cc b/extensions/select_optimization_test.cc index c07f4c6ad..9d4024098 100644 --- a/extensions/select_optimization_test.cc +++ b/extensions/select_optimization_test.cc @@ -254,8 +254,9 @@ class MockAccessApis : public LegacyTypeInfoApis, public LegacyTypeAccessApis { return nullptr; } - absl::optional FindFieldByName( - absl::string_view field_name) const override { + std::optional< + google::api::expr::runtime::LegacyTypeInfoApis::FieldDescription> + FindFieldByName(absl::string_view field_name) const override { return absl::nullopt; } From b5db1b3dffb7b890f1a05110ad5833ebe0ebdbee Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sat, 23 May 2026 23:10:55 -0700 Subject: [PATCH 036/350] No public description PiperOrigin-RevId: 920403526 --- eval/compiler/flat_expr_builder.cc | 10 +++++----- eval/compiler/flat_expr_builder_extensions.cc | 2 +- eval/compiler/qualified_reference_resolver.cc | 10 +++++----- eval/compiler/regex_precompilation_optimization.cc | 6 +++--- eval/compiler/resolver.cc | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index e38c912c0..8558c7007 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -840,7 +840,7 @@ class FlatExprVisitor : public cel::AstVisitor { // Attempt to resolve a select expression as a namespaced identifier for an // enum or type constant value. - absl::optional const_value; + std::optional const_value; int64_t select_root_id = -1; std::string path_candidate; @@ -1080,7 +1080,7 @@ class FlatExprVisitor : public cel::AstVisitor { // Returns the maximum recursion depth of the current program if it is // eligible for recursion, or nullopt if it is not. - absl::optional RecursionEligible() { + std::optional RecursionEligible() { if (!PlanRecursiveProgram() || program_builder_.current() == nullptr) { return absl::nullopt; } @@ -1525,7 +1525,7 @@ class FlatExprVisitor : public cel::AstVisitor { } } } - if (absl::optional depth = RecursionEligible(); depth.has_value()) { + if (std::optional depth = RecursionEligible(); depth.has_value()) { auto deps = ExtractRecursiveDependencies(); if (deps.size() != list_expr.elements().size()) { SetProgressStatusError(absl::InternalError( @@ -1855,7 +1855,7 @@ class FlatExprVisitor : public cel::AstVisitor { int64_t expr_id) { absl::string_view ast_name = create_struct_expr.name(); - absl::optional> type; + std::optional> type; CEL_ASSIGN_OR_RETURN(type, resolver_.FindType(ast_name, expr_id)); if (!type.has_value()) { @@ -1932,7 +1932,7 @@ class FlatExprVisitor : public cel::AstVisitor { IndexManager index_manager_; bool enable_optional_types_; - absl::optional block_; + std::optional block_; int max_recursion_depth_ = 0; }; diff --git a/eval/compiler/flat_expr_builder_extensions.cc b/eval/compiler/flat_expr_builder_extensions.cc index 463b48425..e51b64023 100644 --- a/eval/compiler/flat_expr_builder_extensions.cc +++ b/eval/compiler/flat_expr_builder_extensions.cc @@ -98,7 +98,7 @@ size_t Subexpression::ComputeSize() const { return size; } -absl::optional Subexpression::RecursiveDependencyDepth() const { +std::optional Subexpression::RecursiveDependencyDepth() const { auto* tree = absl::get_if(&program_); int depth = 0; if (tree == nullptr) { diff --git a/eval/compiler/qualified_reference_resolver.cc b/eval/compiler/qualified_reference_resolver.cc index 67f86ebb6..67c14d9b2 100644 --- a/eval/compiler/qualified_reference_resolver.cc +++ b/eval/compiler/qualified_reference_resolver.cc @@ -81,9 +81,9 @@ bool OverloadExists(const Resolver& resolver, absl::string_view name, // Return the qualified name of the most qualified matching overload, or // nullopt if no matches are found. -absl::optional BestOverloadMatch(const Resolver& resolver, - absl::string_view base_name, - int argument_count) { +std::optional BestOverloadMatch(const Resolver& resolver, + absl::string_view base_name, + int argument_count) { if (IsSpecialFunction(base_name)) { return std::string(base_name); } @@ -262,8 +262,8 @@ class ReferenceResolver : public cel::AstRewriterBase { // Convert a select expr sub tree into a namespace name if possible. // If any operand of the top element is a not a select or an ident node, // return nullopt. - absl::optional ToNamespace(const Expr& expr) { - absl::optional maybe_parent_namespace; + std::optional ToNamespace(const Expr& expr) { + std::optional maybe_parent_namespace; if (rewritten_reference_.find(expr.id()) != rewritten_reference_.end()) { // The target expr matches a reference (resolved to an ident decl). // This should not be treated as a function qualifier. diff --git a/eval/compiler/regex_precompilation_optimization.cc b/eval/compiler/regex_precompilation_optimization.cc index b94cae383..455796131 100644 --- a/eval/compiler/regex_precompilation_optimization.cc +++ b/eval/compiler/regex_precompilation_optimization.cc @@ -145,7 +145,7 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { // Try to check if the regex is valid, whether or not we can actually update // the plan. - absl::optional pattern = + std::optional pattern = GetConstantString(context, subexpression, node, pattern_expr); if (!pattern.has_value()) { return absl::OkStatus(); @@ -168,7 +168,7 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { } private: - absl::optional GetConstantString( + std::optional GetConstantString( PlannerContext& context, ProgramBuilder::Subexpression* absl_nullable subexpression, const Expr& call_expr, const Expr& re_expr) const { @@ -180,7 +180,7 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { // Already modified, can't recover the input pattern. return absl::nullopt; } - absl::optional constant; + std::optional constant; if (subexpression->IsRecursive()) { const auto& program = subexpression->recursive_program(); auto deps = program.step->GetDependencies(); diff --git a/eval/compiler/resolver.cc b/eval/compiler/resolver.cc index 4e3fa3841..17f60eaad 100644 --- a/eval/compiler/resolver.cc +++ b/eval/compiler/resolver.cc @@ -102,8 +102,8 @@ absl::Span Resolver::GetPrefixesFor( return namespace_prefixes_; } -absl::optional Resolver::FindConstant(absl::string_view name, - int64_t expr_id) const { +std::optional Resolver::FindConstant(absl::string_view name, + int64_t expr_id) const { auto prefixes = GetPrefixesFor(name); for (const auto& prefix : prefixes) { std::string qualified_name = absl::StrCat(prefix, name); @@ -205,7 +205,7 @@ std::vector Resolver::FindLazyOverloads( return funcs; } -absl::StatusOr>> +absl::StatusOr>> Resolver::FindType(absl::string_view name, int64_t expr_id) const { auto prefixes = GetPrefixesFor(name); for (auto& prefix : prefixes) { From 9911e079ac31ac087e954339170fdd4bc05d3206 Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Tue, 26 May 2026 11:21:49 -0700 Subject: [PATCH 037/350] Map well-known protobuf types to their CEL equivalents in LegacyRuntimeType. PiperOrigin-RevId: 921579763 --- common/BUILD | 1 - common/type.cc | 55 ++++++++++++++++++++++++++++++++++++++------- common/type_test.cc | 34 ++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/common/BUILD b/common/BUILD index ffc4ae1e9..a016d2cb5 100644 --- a/common/BUILD +++ b/common/BUILD @@ -583,7 +583,6 @@ cc_library( "//internal:string_pool", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/common/type.cc b/common/type.cc index f94e8bc52..684c5ba09 100644 --- a/common/type.cc +++ b/common/type.cc @@ -640,8 +640,6 @@ constexpr absl::string_view kUInt64TypeName = "uint"; constexpr absl::string_view kDoubleTypeName = "double"; constexpr absl::string_view kStringTypeName = "string"; constexpr absl::string_view kBytesTypeName = "bytes"; -constexpr absl::string_view kDurationTypeName = "google.protobuf.Duration"; -constexpr absl::string_view kTimestampTypeName = "google.protobuf.Timestamp"; constexpr absl::string_view kListTypeName = "list"; constexpr absl::string_view kMapTypeName = "map"; constexpr absl::string_view kCelTypeTypeName = "type"; @@ -670,12 +668,6 @@ Type LegacyRuntimeType(absl::string_view name) { if (name == kBytesTypeName) { return BytesType{}; } - if (name == kDurationTypeName) { - return DurationType{}; - } - if (name == kTimestampTypeName) { - return TimestampType{}; - } if (name == kListTypeName) { return ListType{}; } @@ -685,6 +677,53 @@ Type LegacyRuntimeType(absl::string_view name) { if (name == kCelTypeTypeName) { return TypeType{}; } + if (cel::IsWellKnownMessageType(name)) { + if (name == "google.protobuf.Any") { + return AnyType(); + } + if (name == "google.protobuf.BoolValue") { + return BoolWrapperType(); + } + if (name == "google.protobuf.BytesValue") { + return BytesWrapperType(); + } + if (name == "google.protobuf.DoubleValue") { + return DoubleWrapperType(); + } + if (name == "google.protobuf.Duration") { + return DurationType(); + } + if (name == "google.protobuf.FloatValue") { + return DoubleWrapperType(); + } + if (name == "google.protobuf.Int32Value") { + return IntWrapperType(); + } + if (name == "google.protobuf.Int64Value") { + return IntWrapperType(); + } + if (name == "google.protobuf.ListValue") { + return ListType(); + } + if (name == "google.protobuf.StringValue") { + return StringWrapperType(); + } + if (name == "google.protobuf.Struct") { + return JsonMapType(); + } + if (name == "google.protobuf.Timestamp") { + return TimestampType(); + } + if (name == "google.protobuf.UInt32Value") { + return UintWrapperType(); + } + if (name == "google.protobuf.UInt64Value") { + return UintWrapperType(); + } + if (name == "google.protobuf.Value") { + return DynType(); + } + } return common_internal::MakeBasicStructType(name); } diff --git a/common/type_test.cc b/common/type_test.cc index 2cebf27ba..d6a613c3c 100644 --- a/common/type_test.cc +++ b/common/type_test.cc @@ -638,5 +638,39 @@ TEST(Type, Wrap) { EXPECT_EQ(Type(AnyType()).Wrap(), AnyType()); } +TEST(Type, LegacyRuntimeType) { + EXPECT_EQ(common_internal::LegacyRuntimeType("bool"), BoolType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Any"), + AnyType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.BoolValue"), + BoolWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.BytesValue"), + BytesWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.DoubleValue"), + DoubleWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Duration"), + DurationType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.FloatValue"), + DoubleWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Int32Value"), + IntWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Int64Value"), + IntWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.ListValue"), + ListType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.StringValue"), + StringWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Struct"), + JsonMapType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Timestamp"), + TimestampType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.UInt32Value"), + UintWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.UInt64Value"), + UintWrapperType()); + EXPECT_EQ(common_internal::LegacyRuntimeType("google.protobuf.Value"), + DynType()); +} + } // namespace } // namespace cel From e8f6c48fd01acc29dc67b8b0d2d6ce68ea36c94c Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 26 May 2026 15:05:42 -0700 Subject: [PATCH 038/350] Repo move announcement PiperOrigin-RevId: 921705653 --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 23afe2b00..41b44388d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # C++ Implementations of the Common Expression Language +> [!WARNING] +> **On June 16, 2026, this repository will move to +> github.com/cel-expr/cel-cpp!** +> +> Please update your links and dependencies. See the [pinned +> issue](https://github.com/google/cel-cpp/issues/2029) for details. + For background on the Common Expression Language see the [cel-spec][1] repo. This is a C++ implementation of a [Common Expression Language][1] runtime, From 2f0944e9f25cbe23b202c280f48f0e47f638668c Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 27 May 2026 10:42:49 -0700 Subject: [PATCH 039/350] Remove constraint that block initializer is non-empty. PiperOrigin-RevId: 922222081 --- eval/compiler/flat_expr_builder.cc | 10 ++++------ eval/compiler/flat_expr_builder_test.cc | 7 +++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 8558c7007..1e3f4ecd3 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -1047,11 +1047,7 @@ class FlatExprVisitor : public cel::AstVisitor { } const auto& list_expr = call_expr.args().front().list_expr(); block.size = list_expr.elements().size(); - if (block.size == 0) { - SetProgressStatusError(absl::InvalidArgumentError( - "malformed cel.@block: list of bound expressions is empty")); - return; - } + block.bindings_set.reserve(block.size); for (const auto& list_expr_element : list_expr.elements()) { if (list_expr_element.optional()) { @@ -2052,7 +2048,9 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleBlock( } // Otherwise, iterative plan. - AddStep(CreateClearSlotsStep(block.index, block.slot_count, expr.id())); + if (block.slot_count > 0) { + AddStep(CreateClearSlotsStep(block.index, block.slot_count, expr.id())); + } return CallHandlerResult::kIntercepted; } diff --git a/eval/compiler/flat_expr_builder_test.cc b/eval/compiler/flat_expr_builder_test.cc index 5fc20f01e..d84007485 100644 --- a/eval/compiler/flat_expr_builder_test.cc +++ b/eval/compiler/flat_expr_builder_test.cc @@ -2820,6 +2820,7 @@ TEST(FlatExprBuilderTest, BlockNotListOfBoundExpressions) { TEST(FlatExprBuilderTest, BlockEmptyListOfBoundExpressions) { ParsedExpr parsed_expr; + // Allowed, but degenerate case. ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( R"pb( expr: { @@ -2835,10 +2836,8 @@ TEST(FlatExprBuilderTest, BlockEmptyListOfBoundExpressions) { CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); EXPECT_THAT( builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()), - StatusIs( - absl::StatusCode::kInvalidArgument, - HasSubstr( - "malformed cel.@block: list of bound expressions is empty"))); + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid @index greater than number of bindings:"))); } TEST(FlatExprBuilderTest, BlockOptionalListOfBoundExpressions) { From 9fb8e1068781eeb988cf5fcd33aaa05a0c8356b6 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 27 May 2026 12:56:46 -0700 Subject: [PATCH 040/350] Return the first match from `ResolveStatic()` PiperOrigin-RevId: 922297232 --- eval/eval/function_step.cc | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index 2a10e9674..fcf429378 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -286,20 +286,12 @@ absl::Status AbstractFunctionStep::Evaluate(ExecutionFrame* frame) const { absl::StatusOr ResolveStatic( absl::Span input_args, absl::Span overloads) { - ResolveResult result = absl::nullopt; - for (const auto& overload : overloads) { if (ArgumentKindsMatch(overload.descriptor, input_args)) { - // More than one overload matches our arguments. - if (result.has_value()) { - return absl::Status(absl::StatusCode::kInternal, - "Cannot resolve overloads"); - } - - result.emplace(overload); + return overload; } } - return result; + return absl::nullopt; } absl::StatusOr ResolveLazy( @@ -315,7 +307,7 @@ absl::StatusOr ResolveLazy( input_args.begin(), input_args.end(), arg_types.begin(), [](const cel::Value& value) { return ValueKindToKind(value->kind()); }); - cel::FunctionDescriptor matcher{name, receiver_style, arg_types}; + cel::FunctionDescriptor matcher{name, receiver_style, std::move(arg_types)}; const cel::ActivationInterface& activation = frame.activation(); for (auto provider : providers) { From 83ce6bd890aef1c2ae41d8179490dd735d2549f2 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 28 May 2026 11:15:52 -0700 Subject: [PATCH 041/350] Add benchmark test for field access implementation. Adds a `cc_test` target for benchmarking `field_access_impl.cc`. PiperOrigin-RevId: 922875336 --- eval/public/structs/BUILD | 20 ++ .../field_access_impl_benchmark_test.cc | 239 ++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 eval/public/structs/field_access_impl_benchmark_test.cc diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index d301ff0ca..d722559e3 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -442,3 +442,23 @@ cc_test( "@com_google_protobuf//:protobuf", ], ) + +cc_test( + name = "field_access_impl_benchmark_test", + srcs = ["field_access_impl_benchmark_test.cc"], + tags = [ + "benchmark", + "manual", + ], + deps = [ + ":cel_proto_wrapper", + ":field_access_impl", + "//eval/public:cel_options", + "//eval/public:cel_value", + "//extensions/protobuf/internal:map_reflection", + "//internal:benchmark", + "//internal:testing", + "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + ], +) diff --git a/eval/public/structs/field_access_impl_benchmark_test.cc b/eval/public/structs/field_access_impl_benchmark_test.cc new file mode 100644 index 000000000..888e424b1 --- /dev/null +++ b/eval/public/structs/field_access_impl_benchmark_test.cc @@ -0,0 +1,239 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "eval/public/cel_options.h" +#include "eval/public/cel_value.h" +#include "eval/public/structs/cel_proto_wrapper.h" +#include "eval/public/structs/field_access_impl.h" +#include "extensions/protobuf/internal/map_reflection.h" +#include "internal/benchmark.h" +#include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/map_field.h" +#include "google/protobuf/message.h" + +namespace google::api::expr::runtime::internal { +namespace { + +using ::cel::expr::conformance::proto3::TestAllTypes; + +void BM_CreateValueFromSingleField_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.set_single_int64(42); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + + for (auto _ : state) { + auto value = CreateValueFromSingleField( + &msg, desc, ProtoWrapperTypeOptions::kUnsetProtoDefault, + &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromSingleField_Int64); + +void BM_CreateValueFromSingleField_String(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.set_single_string("hello world"); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("single_string"); + + for (auto _ : state) { + auto value = CreateValueFromSingleField( + &msg, desc, ProtoWrapperTypeOptions::kUnsetProtoDefault, + &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromSingleField_String); + +void BM_CreateValueFromSingleField_Message(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.mutable_standalone_message()->set_bb(123); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("standalone_message"); + + for (auto _ : state) { + auto value = CreateValueFromSingleField( + &msg, desc, ProtoWrapperTypeOptions::kUnsetProtoDefault, + &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromSingleField_Message); + +void BM_CreateValueFromRepeatedField_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.add_repeated_int64(42); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_int64"); + + for (auto _ : state) { + auto value = CreateValueFromRepeatedField( + &msg, desc, 0, &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromRepeatedField_Int64); + +void BM_CreateValueFromRepeatedField_String(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.add_repeated_string("hello world"); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_string"); + + for (auto _ : state) { + auto value = CreateValueFromRepeatedField( + &msg, desc, 0, &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromRepeatedField_String); + +void BM_CreateValueFromMapValue_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + (*msg.mutable_map_int64_int64())[42] = 100; + const google::protobuf::FieldDescriptor* map_desc = + TestAllTypes::descriptor()->FindFieldByName("map_int64_int64"); + const google::protobuf::FieldDescriptor* value_desc = + map_desc->message_type()->FindFieldByName("value"); + + google::protobuf::ConstMapIterator iter = + cel::extensions::protobuf_internal::ConstMapBegin(*msg.GetReflection(), + msg, *map_desc); + google::protobuf::MapValueConstRef value_ref = iter.GetValueRef(); + + for (auto _ : state) { + auto value = + CreateValueFromMapValue(&msg, value_desc, &value_ref, + &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromMapValue_Int64); + +void BM_SetValueToSingleField_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + CelValue val = CelValue::CreateInt64(42); + + for (auto _ : state) { + auto status = SetValueToSingleField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_SetValueToSingleField_Int64); + +void BM_SetValueToSingleField_String(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("single_string"); + CelValue val = CelValue::CreateStringView("hello world"); + + for (auto _ : state) { + auto status = SetValueToSingleField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_SetValueToSingleField_String); + +void BM_SetValueToSingleField_Message(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("standalone_message"); + + TestAllTypes::NestedMessage nested_msg; + nested_msg.set_bb(123); + CelValue val = CelProtoWrapper::CreateMessage(&nested_msg, &arena); + + for (auto _ : state) { + auto status = SetValueToSingleField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_SetValueToSingleField_Message); + +void BM_AddValueToRepeatedField_Int64(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_int64"); + CelValue val = CelValue::CreateInt64(42); + + for (auto _ : state) { + msg.clear_repeated_int64(); + auto status = AddValueToRepeatedField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_AddValueToRepeatedField_Int64); + +void BM_AddValueToRepeatedField_String(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_string"); + CelValue val = CelValue::CreateStringView("hello world"); + + for (auto _ : state) { + msg.clear_repeated_string(); + auto status = AddValueToRepeatedField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_AddValueToRepeatedField_String); + +void BM_CreateValueFromRepeatedField_StringPiece(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + msg.add_repeated_string_piece("hello world"); + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_string_piece"); + + for (auto _ : state) { + auto value = CreateValueFromRepeatedField( + &msg, desc, 0, &CelProtoWrapper::InternalWrapMessage, &arena); + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_CreateValueFromRepeatedField_StringPiece); + +void BM_AddValueToRepeatedField_StringPiece(benchmark::State& state) { + google::protobuf::Arena arena; + TestAllTypes msg; + const google::protobuf::FieldDescriptor* desc = + TestAllTypes::descriptor()->FindFieldByName("repeated_string_piece"); + CelValue val = CelValue::CreateStringView("hello world"); + + for (auto _ : state) { + msg.clear_repeated_string_piece(); + auto status = AddValueToRepeatedField(val, desc, &msg, &arena); + benchmark::DoNotOptimize(status); + } +} +BENCHMARK(BM_AddValueToRepeatedField_StringPiece); + +} // namespace +} // namespace google::api::expr::runtime::internal From 5436842151a69b1dd3e0c4350c7406b4fba36805 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 1 Jun 2026 12:51:26 -0700 Subject: [PATCH 042/350] Update presubmit docker image. Use bazelisk to simplify testing bazel upgrades. PiperOrigin-RevId: 924861509 --- Dockerfile | 45 +++++++++++++++++++++++++++++---------------- cloudbuild.yaml | 4 ++-- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index c2c2915be..97611fc75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,25 +12,25 @@ # # Run the following command from the root of the CEL repository: # -# gcloud builds submit --region=us -t gcr.io/cel-analysis/gcc9 . +# gcloud builds submit --region=us -t gcr.io/cel-analysis/cel-cpp/ubuntu_floor . # # Once complete get the sha256 digest from the output using the following # command: # -# gcloud artifacts versions list --package=gcc9 --repository=gcr.io \ +# gcloud artifacts versions list --package=cel-cpp/ubuntu_floor --repository=gcr.io \ # --location=us # # The cloudbuild.yaml file must be updated to use the new digest like so: # -# - name: 'gcr.io/cel-analysis/gcc9@' -FROM gcc:9 +# - name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@' +FROM gcr.io/cloud-marketplace/google/ubuntu2204:latest # Install Bazel prerequesites and required tools. # See https://docs.bazel.build/versions/master/install-ubuntu.html -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ git \ libssl-dev \ make \ @@ -41,16 +41,29 @@ RUN apt-get update && \ zip \ zlib1g-dev \ default-jdk-headless \ - clang-11 && \ - apt-get clean + clang-11 \ + gcc-9 g++-9 \ + tzdata \ + && apt-get clean -# Install Bazel. -# https://github.com/bazelbuild/bazel/releases -ARG BAZEL_VERSION="7.3.2" -ADD https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh /tmp/install_bazel.sh -RUN /bin/bash /tmp/install_bazel.sh && rm /tmp/install_bazel.sh +# Install Bazelisk. +# https://github.com/bazelbuild/bazelisk/releases +ARG BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/v1.27.0/bazelisk-amd64.deb" +ARG BAZELISK_CHKSUM="d8b00ea975c823e15263c80200ac42979e17368547fbff4ab177af035badfa83" +ADD ${BAZELISK_URL} /tmp/bazelisk.deb + +ENV BAZELISK_CHKSUM=${BAZELISK_CHKSUM} +RUN echo "${BAZELISK_CHKSUM} */tmp/bazelisk.deb" | sha256sum --check + +RUN apt-get install /tmp/bazelisk.deb RUN mkdir -p /workspace RUN mkdir -p /bazel -ENTRYPOINT ["/usr/local/bin/bazel"] +RUN USE_BAZEL_VERSION=8.7.0 bazelisk help +RUN USE_BAZEL_VERSION=7.3.2 bazelisk help + +ENV CC=gcc-9 +ENV CXX=g++-9 + +ENTRYPOINT ["/usr/bin/bazelisk"] diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 8272378f6..dec359f25 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,5 +1,5 @@ steps: -- name: 'gcr.io/cel-analysis/gcc9@sha256:4d5ff2e55224398807235a44b57e9c5793e922ac46e9ff428536bb8f8e5790ce' +- name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@sha256:211a0c505b361d987b3d8b08a5144a84e62cb95edc3f897fe46d5cd3f556f79d' args: - '--output_base=/bazel' # This is mandatory to avoid steps accidently sharing data. - 'test' @@ -16,7 +16,7 @@ steps: - '--google_default_credentials' id: gcc-9 waitFor: ['-'] -- name: 'gcr.io/cel-analysis/gcc9@sha256:4d5ff2e55224398807235a44b57e9c5793e922ac46e9ff428536bb8f8e5790ce' +- name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@sha256:211a0c505b361d987b3d8b08a5144a84e62cb95edc3f897fe46d5cd3f556f79d' env: - 'CC=clang-11' - 'CXX=clang++-11' From f1e5042c97e641457b92dcd4cdee57d1facf96fa Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 1 Jun 2026 14:06:53 -0700 Subject: [PATCH 043/350] Bump bazel version to 8.x PiperOrigin-RevId: 924902706 --- .bazelversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelversion b/.bazelversion index eab246c06..df5119ec6 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -7.3.2 +8.7.0 From dfea16a3fefaace88eafbdb294bc8ea376bb1c9b Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 1 Jun 2026 14:38:20 -0700 Subject: [PATCH 044/350] Update deps. Patches to antlr were upstreamed to bcr. PiperOrigin-RevId: 924919565 --- MODULE.bazel | 17 ++++------------- bazel/antlr.patch | 30 ------------------------------ 2 files changed, 4 insertions(+), 43 deletions(-) delete mode 100644 bazel/antlr.patch diff --git a/MODULE.bazel b/MODULE.bazel index 02404b645..43d0485d2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -33,7 +33,7 @@ bazel_dep( ) bazel_dep( name = "protobuf", - version = "33.4", + version = "34.1", repo_name = "com_google_protobuf", ) bazel_dep( @@ -41,20 +41,16 @@ bazel_dep( version = "20260107.0", repo_name = "com_google_absl", ) - bazel_dep( name = "googletest", version = "1.17.0.bcr.2", - dev_dependency = True, repo_name = "com_google_googletest", ) bazel_dep( name = "google_benchmark", version = "1.9.2", - dev_dependency = True, repo_name = "com_github_google_benchmark", ) - bazel_dep( name = "re2", version = "2025-11-05.bcr.1", @@ -74,16 +70,9 @@ bazel_dep( name = "platforms", version = "1.0.0", ) - -ANTLR4_VERSION = "4.13.2" - bazel_dep( name = "antlr4-cpp-runtime", - version = ANTLR4_VERSION, -) -single_version_override( - module_name = "antlr4-cpp-runtime", - patches = ["//bazel:antlr.patch"], + version = "4.13.2.bcr.2", ) python = use_extension("@rules_python//python/extensions:python.bzl", "python") @@ -95,6 +84,8 @@ python.toolchain( http_jar = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") +ANTLR4_VERSION = "4.13.2" + http_jar( name = "antlr4_jar", sha256 = "eae2dfa119a64327444672aff63e9ec35a20180dc5b8090b7a6ab85125df4d76", diff --git a/bazel/antlr.patch b/bazel/antlr.patch deleted file mode 100644 index c1aa9080c..000000000 --- a/bazel/antlr.patch +++ /dev/null @@ -1,30 +0,0 @@ ---- BUILD.bazel -+++ BUILD.bazel -@@ -17,21 +17,21 @@ - cc_library( - name = "antlr4-cpp-runtime", - srcs = glob(["runtime/src/**/*.cpp"]), - hdrs = ["runtime/src/antlr4-runtime.h"], - copts = ["-fexceptions"], -- defines = ["ANTLR4CPP_USING_ABSEIL"], -+ defines = ["ANTLR4CPP_USING_ABSEIL", "ANTLR4CPP_STATIC"], - features = ["-use_header_modules"], - includes = ["runtime/src"], - textual_hdrs = glob( - ["runtime/src/**/*.h"], - exclude = ["runtime/src/antlr4-runtime.h"], - ), - visibility = ["//visibility:public"], - deps = [ - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/synchronization", - ], - ) - ---- VERSION -+++ /dev/null -@@ -1,1 +1,0 @@ --4.13.2 \ No newline at end of file From c2ae232042a0c37fc9bc2674daa73648f48ae86f Mon Sep 17 00:00:00 2001 From: sahvx655-wq Date: Tue, 2 Jun 2026 11:58:15 +0530 Subject: [PATCH 045/350] fix code unit/code point mix-up in cord substring length --- common/values/string_value.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/values/string_value.cc b/common/values/string_value.cc index 98912d32c..c14756bb2 100644 --- a/common/values/string_value.cc +++ b/common/values/string_value.cc @@ -742,7 +742,7 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start, } if (size_code_points == end) { return cord.Subcord(start_code_units, - size_code_points - start_code_units); + size_code_units - start_code_units); } char32_t code_point; size_t code_units; From db0725c120fab156cedf32776bba9d35d9406d85 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 2 Jun 2026 13:31:45 -0700 Subject: [PATCH 046/350] Add basic workflow for windows/bazel test (#2039) Add basic workflow for running conformance tests on windows. Only run on post merge or manually for now. Closes #2039 PiperOrigin-RevId: 925547950 --- .bazelrc | 2 + .github/workflows/windows_bazel_test.yml | 28 +++++++++++++ .../windows_bazel_test_post_merge.yml | 13 ++++++ conformance/BUILD | 1 + conformance/run.bzl | 2 +- conformance/run.cc | 7 ++-- internal/BUILD | 11 +++++ internal/runfiles.cc | 40 +++++++++++++++++++ internal/runfiles.h | 30 ++++++++++++++ 9 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/windows_bazel_test.yml create mode 100644 .github/workflows/windows_bazel_test_post_merge.yml create mode 100644 internal/runfiles.cc create mode 100644 internal/runfiles.h diff --git a/.bazelrc b/.bazelrc index 475706072..1246d336b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -10,6 +10,8 @@ build:linux --copt=-Wno-deprecated-declarations # you will typically need to spell out the compiler for local dev # BAZEL_VC= # BAZEL_VC_FULL_VERSION=14.44.3520 +# Some dependencies rely on bash so you will likely need msys2 +# BAZEL_SH=C:\msys64\usr\bin\bash.exe build:msvc --cxxopt="-std:c++20" --cxxopt="-utf-8" --host_cxxopt="-std:c++20" build:msvc --define=protobuf_allow_msvc=true build:msvc --test_tag_filters=-benchmark,-notap,-no_test_msvc diff --git a/.github/workflows/windows_bazel_test.yml b/.github/workflows/windows_bazel_test.yml new file mode 100644 index 000000000..4ac7f2eec --- /dev/null +++ b/.github/workflows/windows_bazel_test.yml @@ -0,0 +1,28 @@ +name: Windows Bazel Test + +on: + workflow_call: + workflow_dispatch: + +jobs: + test: + name: Run Bazel Tests + runs-on: windows-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Bazel and Bazelisk + uses: bazel-contrib/setup-bazel@0.19.0 + with: + bazelisk-cache: true + disk-cache: ${{ github.workflow }} + repository-cache: true + + - name: Run Tests + # msys2 'bash' on Windows will try to 'fix' the label prefix to + # work as a directory. + # //... won't work. + shell: bash + run: | + bazelisk test --config=msvc conformance:all \ No newline at end of file diff --git a/.github/workflows/windows_bazel_test_post_merge.yml b/.github/workflows/windows_bazel_test_post_merge.yml new file mode 100644 index 000000000..11801011e --- /dev/null +++ b/.github/workflows/windows_bazel_test_post_merge.yml @@ -0,0 +1,13 @@ +name: Windows Bazel Test (Post-Merge) + +on: + push: + branches: + - master + +jobs: + trigger-test: + # This prevents the workflow from running automatically when someone + # pushes to their fork. + if: github.repository == 'google/cel-cpp' + uses: ./.github/workflows/windows_bazel_test.yml \ No newline at end of file diff --git a/conformance/BUILD b/conformance/BUILD index 0ca90a4bc..a6f25e001 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -99,6 +99,7 @@ cc_library( deps = [ ":service", ":utils", + "//internal:runfiles", "//internal:testing_no_main", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/log:absl_check", diff --git a/conformance/run.bzl b/conformance/run.bzl index 15850b0aa..2c0b51c0e 100644 --- a/conformance/run.bzl +++ b/conformance/run.bzl @@ -77,7 +77,7 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard): cc_test( name = _conformance_test_name(name, optimize, recursive), - args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard) + ["$(location " + test + ")" for test in data], + args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard) + ["$(rlocationpath {})".format(test) for test in data], env = select( { "@platforms//os:windows": {"CEL_SKIP_TESTS": ",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS)}, diff --git a/conformance/run.cc b/conformance/run.cc index 80164d9a4..4a0493494 100644 --- a/conformance/run.cc +++ b/conformance/run.cc @@ -48,6 +48,7 @@ #include "absl/types/span.h" #include "conformance/service.h" #include "conformance/utils.h" +#include "internal/runfiles.h" #include "internal/testing.h" #include "cel/expr/conformance/test/simple.pb.h" #include "google/protobuf/io/zero_copy_stream_impl.h" @@ -68,8 +69,6 @@ ABSL_FLAG(bool, select_optimization, false, "Enable select optimization."); namespace { -using ::testing::IsEmpty; - using cel::expr::conformance::test::SimpleTest; using cel::expr::conformance::test::SimpleTestFile; using google::api::expr::conformance::v1alpha1::CheckRequest; @@ -78,6 +77,7 @@ using google::api::expr::conformance::v1alpha1::EvalRequest; using google::api::expr::conformance::v1alpha1::EvalResponse; using google::api::expr::conformance::v1alpha1::ParseRequest; using google::api::expr::conformance::v1alpha1::ParseResponse; +using ::testing::IsEmpty; google::rpc::Code ToGrpcCode(absl::StatusCode code) { return static_cast(code); @@ -282,8 +282,9 @@ int main(int argc, char** argv) { } } for (int argi = 1; argi < argc; argi++) { + std::string path = cel::internal::ResolveRunfilesPath(argv[argi]); ABSL_CHECK_OK(RegisterTestsFromFile(service, tests_to_skip, - absl::string_view(argv[argi]))); + absl::string_view(path))); } } int exit_code = RUN_ALL_TESTS(); diff --git a/internal/BUILD b/internal/BUILD index 3891c635d..0ac5c4e46 100644 --- a/internal/BUILD +++ b/internal/BUILD @@ -86,6 +86,17 @@ cc_library( ], ) +cc_library( + name = "runfiles", + srcs = ["runfiles.cc"], + hdrs = ["runfiles.h"], + deps = [ + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + "@rules_cc//cc/runfiles", + ], +) + cc_library( name = "status_builder", hdrs = ["status_builder.h"], diff --git a/internal/runfiles.cc b/internal/runfiles.cc new file mode 100644 index 000000000..259e2e7ca --- /dev/null +++ b/internal/runfiles.cc @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/runfiles.h" + +#include + +#include "rules_cc/cc/runfiles/runfiles.h" + +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace cel::internal { + +std::string ResolveRunfilesPath(absl::string_view path) { + using ::rules_cc::cc::runfiles::Runfiles; + static Runfiles* runfiles = []() { + std::string error; + auto runfiles = + Runfiles::CreateForTest(BAZEL_CURRENT_REPOSITORY, &error); + ABSL_QCHECK(runfiles != nullptr) + << absl::StrCat("failed to init runfiles", error); + return runfiles; + }(); + return runfiles->Rlocation(std::string(path)); +} + +} // namespace cel::internal diff --git a/internal/runfiles.h b/internal/runfiles.h new file mode 100644 index 000000000..643c677b4 --- /dev/null +++ b/internal/runfiles.h @@ -0,0 +1,30 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ +#define THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ + +#include + +#include "absl/strings/string_view.h" + +namespace cel::internal { + +// Resolves a path relative to the runfiles directory. +// Intended for resolving test cases from cel-spec and cel-policy. +std::string ResolveRunfilesPath(absl::string_view path); + +} // namespace cel::internal + +#endif // THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ From e264932db519034136ba38cb7381405b32ee9da6 Mon Sep 17 00:00:00 2001 From: sahvx655-wq Date: Wed, 3 Jun 2026 23:04:15 +0530 Subject: [PATCH 047/350] fix undefined left shift of negative int in math.bitShiftLeft --- extensions/math_ext.cc | 6 +++++- extensions/math_ext_test.cc | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/extensions/math_ext.cc b/extensions/math_ext.cc index 4d133d90c..a7773da19 100644 --- a/extensions/math_ext.cc +++ b/extensions/math_ext.cc @@ -266,7 +266,11 @@ Value BitShiftLeftInt(int64_t lhs, int64_t rhs) { if (rhs > 63) { return IntValue(0); } - return IntValue(lhs << static_cast(rhs)); + // Shift in the unsigned domain to avoid undefined behaviour when lhs is + // negative or the shift moves bits into the sign bit, matching the bit + // pattern semantics already used by bitShiftRight. + return IntValue(absl::bit_cast(absl::bit_cast(lhs) + << static_cast(rhs))); } Value BitShiftLeftUint(uint64_t lhs, int64_t rhs) { diff --git a/extensions/math_ext_test.cc b/extensions/math_ext_test.cc index 72605648f..ea9331970 100644 --- a/extensions/math_ext_test.cc +++ b/extensions/math_ext_test.cc @@ -563,6 +563,8 @@ INSTANTIATE_TEST_SUITE_P( {"math.bitNot(2) == -3"}, {"math.bitAnd(math.bitNot(0x3u), 0xFFu) == 0xFCu"}, {"math.bitShiftLeft(1, 1) == 2"}, + {"math.bitShiftLeft(-1, 1) == -2"}, + {"math.bitShiftLeft(-4, 2) == -16"}, {"math.bitShiftLeft(1u, 1) == 2u"}, {"math.bitShiftRight(4, 1) == 2"}, {"math.bitShiftRight(4u, 1) == 2u"}})); From ad137617e146bf2d050747ca3a7fedbcef0e4018 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 3 Jun 2026 14:00:23 -0700 Subject: [PATCH 048/350] Add memory test for managed constant folding arena PiperOrigin-RevId: 926250668 --- eval/tests/memory_safety_test.cc | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/eval/tests/memory_safety_test.cc b/eval/tests/memory_safety_test.cc index 9c0a683e4..a88844fed 100644 --- a/eval/tests/memory_safety_test.cc +++ b/eval/tests/memory_safety_test.cc @@ -51,7 +51,12 @@ struct TestCase { bool reference_resolver_enabled = false; }; -enum Options { kDefault, kExhaustive, kFoldConstants }; +enum Options { + kDefault, + kExhaustive, + kFoldConstants, + kFoldConstantsManagedArena +}; using ParamType = std::tuple; @@ -68,6 +73,9 @@ std::string TestCaseName(const testing::TestParamInfo& param_info) { case Options::kFoldConstants: opt = "opt"; break; + case Options::kFoldConstantsManagedArena: + opt = "opt_managed_arena"; + break; } return absl::StrCat(std::get<0>(param).name, "_", opt); @@ -110,6 +118,14 @@ class EvaluatorMemorySafetyTest : public testing::TestWithParam { options.enable_comprehension_vulnerability_check = false; options.short_circuiting = true; break; + case Options::kFoldConstantsManagedArena: + options.enable_regex_precompilation = true; + options.constant_folding = true; + options.enable_comprehension_list_append = true; + options.enable_comprehension_vulnerability_check = false; + options.short_circuiting = true; + options.constant_arena = nullptr; + break; } options.enable_qualified_identifier_rewrites = @@ -295,7 +311,8 @@ INSTANTIATE_TEST_SUITE_P( test::IsCelBool(true), }}), testing::Values(Options::kDefault, Options::kExhaustive, - Options::kFoldConstants)), + Options::kFoldConstants, + Options::kFoldConstantsManagedArena)), &TestCaseName); } // namespace From f26fd7d9ddf6cace625627e9b6d94d40507e823a Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 4 Jun 2026 11:19:41 -0700 Subject: [PATCH 049/350] Fix leak for optional.of(string). PiperOrigin-RevId: 926790246 --- common/values/optional_value.cc | 4 ++-- runtime/BUILD | 1 + runtime/memory_safety_test.cc | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/common/values/optional_value.cc b/common/values/optional_value.cc index 688cf8fb0..7c214b9cb 100644 --- a/common/values/optional_value.cc +++ b/common/values/optional_value.cc @@ -345,7 +345,7 @@ OpaqueValue GenericOptionalValueClone( cel::Value* absl_nonnull result = ::new (arena->AllocateAligned(sizeof(cel::Value), alignof(cel::Value))) cel::Value(content.To().value->Clone(arena)); - if (!ArenaTraits<>::trivially_destructible(result)) { + if (!ArenaTraits<>::trivially_destructible(*result)) { arena->OwnDestructor(result); } return common_internal::MakeOptionalValue( @@ -395,7 +395,7 @@ OptionalValue OptionalValue::Of(cel::Value value, cel::Value* absl_nonnull result = ::new ( arena->AllocateAligned(sizeof(cel::Value), alignof(cel::Value))) cel::Value(std::move(value)); - if (!ArenaTraits<>::trivially_destructible(result)) { + if (!ArenaTraits<>::trivially_destructible(*result)) { arena->OwnDestructor(result); } return OptionalValue(&optional_value_dispatcher, diff --git a/runtime/BUILD b/runtime/BUILD index 776a8223d..34ff411a1 100644 --- a/runtime/BUILD +++ b/runtime/BUILD @@ -615,6 +615,7 @@ cc_test( ":activation", ":constant_folding", ":function_adapter", + ":optional_types", ":reference_resolver", ":regex_precompilation", ":runtime", diff --git a/runtime/memory_safety_test.cc b/runtime/memory_safety_test.cc index 2a09be666..a60b4ce60 100644 --- a/runtime/memory_safety_test.cc +++ b/runtime/memory_safety_test.cc @@ -45,6 +45,7 @@ #include "runtime/activation.h" #include "runtime/constant_folding.h" #include "runtime/function_adapter.h" +#include "runtime/optional_types.h" #include "runtime/reference_resolver.h" #include "runtime/regex_precompilation.h" #include "runtime/runtime.h" @@ -174,6 +175,7 @@ absl::StatusOr> ConfigureRuntimeImpl( if (resolve_references) { CEL_RETURN_IF_ERROR(EnableReferenceResolver( runtime_builder, ReferenceResolverEnabled::kAlways)); + CEL_RETURN_IF_ERROR(extensions::EnableOptionalTypes(runtime_builder)); } if (evaluation_options == Options::kFoldConstants) { CEL_RETURN_IF_ERROR(extensions::EnableConstantFolding(runtime_builder)); @@ -315,6 +317,14 @@ INSTANTIATE_TEST_SUITE_P( {{"condition", BoolValue(false)}}, test::StringValueIs("long_right_hand_string_0123456789"), }, + {"optional_of_long_const_string", + "condition ? optional.of('lhs_short') : " + "optional.of('long_right_hand_string_0123456789')", + {{"condition", BoolValue(false)}}, + test::OptionalValueIs( + test::StringValueIs("long_right_hand_string_0123456789")), + // optional.of is a namespaced function. + /*enable_reference_resolver=*/true}, { "computed_string", "(condition ? 'a.b' : 'b.c') + '.d.e.f'", From e4ff771e68f58433ca20f4471c4a5c9dd86923eb Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 4 Jun 2026 15:39:42 -0700 Subject: [PATCH 050/350] Add ProtoTypeMaskRegistry, ProtoTypeMask, and FieldPath classes. The ProtoTypeMaskRegistry has functions that can be used to validate the input field masks and to check whether a field is visible. PiperOrigin-RevId: 926925505 --- checker/internal/BUILD | 92 ++++ checker/internal/field_path.cc | 30 ++ checker/internal/field_path.h | 77 ++++ checker/internal/field_path_test.cc | 85 ++++ checker/internal/proto_type_mask.cc | 87 ++++ checker/internal/proto_type_mask.h | 111 +++++ checker/internal/proto_type_mask_registry.cc | 180 ++++++++ checker/internal/proto_type_mask_registry.h | 83 ++++ .../internal/proto_type_mask_registry_test.cc | 402 ++++++++++++++++++ checker/internal/proto_type_mask_test.cc | 143 +++++++ 10 files changed, 1290 insertions(+) create mode 100644 checker/internal/field_path.cc create mode 100644 checker/internal/field_path.h create mode 100644 checker/internal/field_path_test.cc create mode 100644 checker/internal/proto_type_mask.cc create mode 100644 checker/internal/proto_type_mask.h create mode 100644 checker/internal/proto_type_mask_registry.cc create mode 100644 checker/internal/proto_type_mask_registry.h create mode 100644 checker/internal/proto_type_mask_registry_test.cc create mode 100644 checker/internal/proto_type_mask_test.cc diff --git a/checker/internal/BUILD b/checker/internal/BUILD index f4c60f937..25550616a 100644 --- a/checker/internal/BUILD +++ b/checker/internal/BUILD @@ -310,3 +310,95 @@ cc_test( "@com_google_absl//absl/types:optional", ], ) + +cc_library( + name = "field_path", + srcs = ["field_path.cc"], + hdrs = ["field_path.h"], + deps = [ + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "field_path_test", + srcs = ["field_path_test.cc"], + deps = [ + ":field_path", + "//internal:testing", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "proto_type_mask", + srcs = ["proto_type_mask.cc"], + hdrs = ["proto_type_mask.h"], + deps = [ + ":field_path", + "//internal:status_macros", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "proto_type_mask_test", + srcs = ["proto_type_mask_test.cc"], + deps = [ + ":field_path", + ":proto_type_mask", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "proto_type_mask_registry", + srcs = ["proto_type_mask_registry.cc"], + hdrs = ["proto_type_mask_registry.h"], + deps = [ + ":field_path", + ":proto_type_mask", + "//common:type", + "//internal:status_macros", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "proto_type_mask_registry_test", + srcs = ["proto_type_mask_registry_test.cc"], + deps = [ + ":proto_type_mask", + ":proto_type_mask_registry", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/checker/internal/field_path.cc b/checker/internal/field_path.cc new file mode 100644 index 000000000..5ecc4219b --- /dev/null +++ b/checker/internal/field_path.cc @@ -0,0 +1,30 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/field_path.h" + +#include + +#include "absl/strings/str_join.h" +#include "absl/strings/substitute.h" + +namespace cel::checker_internal { + +std::string FieldPath::DebugString() const { + return absl::Substitute( + "FieldPath { field path: '$0', field selection: {'$1'} }", path_, + absl::StrJoin(field_selection_, "', '")); +} + +} // namespace cel::checker_internal diff --git a/checker/internal/field_path.h b/checker/internal/field_path.h new file mode 100644 index 000000000..d67d9b935 --- /dev/null +++ b/checker/internal/field_path.h @@ -0,0 +1,77 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_CHECKER_FIELD_PATH_H_ +#define THIRD_PARTY_CEL_CPP_CHECKER_FIELD_PATH_H_ + +#include +#include +#include + +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace cel::checker_internal { + +// Represents a single path within a FieldMask. +class FieldPath { + public: + explicit FieldPath(std::string path) + : path_(std::move(path)), + field_selection_(absl::StrSplit(path_, kPathDelimiter)) {} + + // Returns the input path. + // For example: "f.b.d". + absl::string_view GetPath() const { return path_; } + + // Returns the list of nested field names in the path. + // For example: {"f", "b", "d"}. + absl::Span GetFieldSelection() const { + return field_selection_; + } + + // Returns the first field name in the path. + // For example: "f". + std::string GetFieldName() const { return field_selection_.front(); } + + template + friend void AbslStringify(Sink& sink, const FieldPath& field_path) { + sink.Append(field_path.DebugString()); + } + + private: + static constexpr char kPathDelimiter = '.'; + + std::string DebugString() const; + + // The input path. For example: "f.b.d". + std::string path_; + // The list of nested field names in the path. For example: {"f", "b", "d"}. + std::vector field_selection_; +}; + +inline bool operator==(const FieldPath& lhs, const FieldPath& rhs) { + return lhs.GetFieldSelection() == rhs.GetFieldSelection(); +} + +// Compares the field selections in the field paths. +// This is only intended as an arbitrary ordering for a set. +inline bool operator<(const FieldPath& lhs, const FieldPath& rhs) { + return lhs.GetFieldSelection() < rhs.GetFieldSelection(); +} + +} // namespace cel::checker_internal + +#endif // THIRD_PARTY_CEL_CPP_CHECKER_FIELD_PATH_H_ diff --git a/checker/internal/field_path_test.cc b/checker/internal/field_path_test.cc new file mode 100644 index 000000000..9a1434954 --- /dev/null +++ b/checker/internal/field_path_test.cc @@ -0,0 +1,85 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/field_path.h" + +#include "absl/strings/str_cat.h" +#include "internal/testing.h" + +namespace cel::checker_internal { +namespace { + +using ::testing::ElementsAre; + +TEST(FieldPathTest, EmptyPathReturnsEmptyString) { + FieldPath field_path(""); + EXPECT_EQ(field_path.GetPath(), ""); + EXPECT_THAT(field_path.GetFieldSelection(), ElementsAre("")); + EXPECT_EQ(field_path.GetFieldName(), ""); +} + +TEST(FieldPathTest, DelimiterPathReturnsEmptyStrings) { + FieldPath field_path("."); + EXPECT_EQ(field_path.GetPath(), "."); + EXPECT_THAT(field_path.GetFieldSelection(), ElementsAre("", "")); + EXPECT_EQ(field_path.GetFieldName(), ""); +} + +TEST(FieldPathTest, FieldPathReturnsFields) { + FieldPath field_path("resource.name.other_field"); + EXPECT_EQ(field_path.GetPath(), "resource.name.other_field"); + EXPECT_THAT(field_path.GetFieldSelection(), + ElementsAre("resource", "name", "other_field")); + EXPECT_EQ(field_path.GetFieldName(), "resource"); +} + +TEST(FieldPathTest, AbslStringifyPrintsFieldSelection) { + FieldPath field_path("resource.name"); + EXPECT_EQ(absl::StrCat(field_path), + "FieldPath { field path: 'resource.name', field selection: " + "{'resource', 'name'} }"); +} + +TEST(FieldPathTest, EqualsComparesFieldSelectionAndReturnsTrue) { + FieldPath field_path_1("resource.name"); + FieldPath field_path_2("resource.name"); + EXPECT_TRUE(field_path_1 == field_path_2); +} + +TEST(FieldPathTest, EqualsComparesFieldSelectionAndReturnsFalse) { + FieldPath field_path_1("resource.name"); + FieldPath field_path_2("resource.type"); + EXPECT_FALSE(field_path_1 == field_path_2); +} + +TEST(FieldPathTest, LessThanComparesFieldSelectionAndReturnsTrue) { + FieldPath field_path_1("resource.name"); + FieldPath field_path_2("resource.type"); + EXPECT_TRUE(field_path_1 < field_path_2); +} + +TEST(FieldPathTest, LessThanComparesIdenticalFieldSelectionAndReturnsFalse) { + FieldPath field_path_1("resource.name"); + FieldPath field_path_2("resource.name"); + EXPECT_FALSE(field_path_1 < field_path_2); +} + +TEST(FieldPathTest, LessThanComparesFieldSelectionAndReturnsFalse) { + FieldPath field_path_1("resource.type"); + FieldPath field_path_2("resource.name"); + EXPECT_FALSE(field_path_1 < field_path_2); +} + +} // namespace +} // namespace cel::checker_internal diff --git a/checker/internal/proto_type_mask.cc b/checker/internal/proto_type_mask.cc new file mode 100644 index 000000000..85e39cb69 --- /dev/null +++ b/checker/internal/proto_type_mask.cc @@ -0,0 +1,87 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/proto_type_mask.h" + +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/btree_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "checker/internal/field_path.h" +#include "internal/status_macros.h" +#include "google/protobuf/descriptor.h" + +namespace cel::checker_internal { + +using ::google::protobuf::Descriptor; +using ::google::protobuf::DescriptorPool; +using ::google::protobuf::FieldDescriptor; + +absl::StatusOr FindMessage( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + absl::string_view type_name) { + const Descriptor* descriptor = + descriptor_pool->FindMessageTypeByName(type_name); + if (descriptor == nullptr) { + return absl::InvalidArgumentError( + absl::Substitute("type '$0' not found", type_name)); + } + return descriptor; +} + +absl::StatusOr FindField(const Descriptor* descriptor, + absl::string_view field_name) { + const FieldDescriptor* field_descriptor = + descriptor->FindFieldByName(field_name); + if (field_descriptor == nullptr) { + return absl::InvalidArgumentError( + absl::Substitute("could not select field '$0' from type '$1'", + field_name, descriptor->full_name())); + } + return field_descriptor; +} + +absl::StatusOr> ProtoTypeMask::GetFieldNames( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool) const { + CEL_ASSIGN_OR_RETURN(const Descriptor* descriptor, + FindMessage(descriptor_pool, this->GetTypeName())); + absl::btree_set field_names; + for (const FieldPath& field_path : this->GetFieldPaths()) { + std::string field_name = field_path.GetFieldName(); + CEL_ASSIGN_OR_RETURN(const FieldDescriptor* field_descriptor, + FindField(descriptor, field_name)); + field_names.insert(field_descriptor->name()); + } + return field_names; +} + +std::string ProtoTypeMask::DebugString() const { + // Represent each FieldPath by its path because it is easiest to read. + std::vector paths; + paths.reserve(field_paths_.size()); + for (const FieldPath& field_path : field_paths_) { + paths.emplace_back(field_path.GetPath()); + } + return absl::Substitute( + "ProtoTypeMask { type name: '$0', field paths: { '$1' } }", type_name_, + absl::StrJoin(paths, "', '")); +} + +} // namespace cel::checker_internal diff --git a/checker/internal/proto_type_mask.h b/checker/internal/proto_type_mask.h new file mode 100644 index 000000000..f7d522cba --- /dev/null +++ b/checker/internal/proto_type_mask.h @@ -0,0 +1,111 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_H_ +#define THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_H_ + +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/btree_set.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "checker/internal/field_path.h" +#include "google/protobuf/descriptor.h" + +namespace cel::checker_internal { + +// Returns a descriptor for the input type name. +// Returns an error if the type name is not found. +absl::StatusOr FindMessage( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + absl::string_view type_name); + +// Returns a field descriptor for the input field name. +// Returns an error if the field name is not found. +absl::StatusOr FindField( + const google::protobuf::Descriptor* descriptor, absl::string_view field_name); + +// Represents the fraction of a protobuf type's object graph that should be +// visible within CEL expressions. +class ProtoTypeMask { + public: + explicit ProtoTypeMask(std::string type_name, + const std::vector& field_paths) + : type_name_(std::move(type_name)) { + for (const std::string& field_path : field_paths) { + field_paths_.insert(FieldPath(field_path)); + } + } + + // Returns a set of field names. The set includes the first field name from + // each field path. We are able to return a set of absl::string_view because + // the result is backed by the descriptor pool. + absl::StatusOr> GetFieldNames( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool) const; + + // Returns the type's full name. + // For example: "google.rpc.context.AttributeContext". + absl::string_view GetTypeName() const { return type_name_; } + + // Returns a representation of the FieldMask, which is a set of field paths. + // For example: + // { + // FieldPath { + // field path: 'resource.name', + // field selection: {'resource', 'name'} + // }, + // FieldPath { + // field path: 'request.auth.claims', + // field selection: {'request', 'auth', 'claims'} + // } + // } + const absl::btree_set& GetFieldPaths() const { + return field_paths_; + } + + template + friend void AbslStringify(Sink& sink, const ProtoTypeMask& proto_type_mask) { + sink.Append(proto_type_mask.DebugString()); + } + + private: + std::string DebugString() const; + + // A type's full name. For example: "google.rpc.context.AttributeContext". + std::string type_name_; + // A representation of a FieldMask, which is a set of field paths. + // For example: + // { + // FieldPath { + // field path: 'resource.name', + // field selection: {'resource', 'name'} + // }, + // FieldPath { + // field path: 'request.auth.claims', + // field selection: {'request', 'auth', 'claims'} + // } + // } + // A FieldMask contains one or more paths which contain identifier characters + // that have been dot delimited, e.g. resource.name, request.auth.claims. + // For each path, all descendent fields after the last element in the path are + // visible. An empty set means all fields are hidden. + absl::btree_set field_paths_; +}; + +} // namespace cel::checker_internal + +#endif // THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_H_ diff --git a/checker/internal/proto_type_mask_registry.cc b/checker/internal/proto_type_mask_registry.cc new file mode 100644 index 000000000..9c50c9784 --- /dev/null +++ b/checker/internal/proto_type_mask_registry.cc @@ -0,0 +1,180 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/proto_type_mask_registry.h" + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/btree_set.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "checker/internal/field_path.h" +#include "checker/internal/proto_type_mask.h" +#include "common/type.h" +#include "internal/status_macros.h" +#include "google/protobuf/descriptor.h" + +namespace cel::checker_internal { +namespace { + +using ::google::protobuf::Descriptor; +using ::google::protobuf::DescriptorPool; +using ::google::protobuf::FieldDescriptor; +using TypeMap = + absl::flat_hash_map>; + +// Returns a message type descriptor for the input field descriptor. +// Returns an error if the field is not a message type. +absl::StatusOr GetMessage( + const FieldDescriptor* field_descriptor) { + cel::MessageTypeField field(field_descriptor); + cel::Type type = field.GetType(); + absl::optional message_type = type.AsMessage(); + if (!message_type.has_value()) { + return absl::InvalidArgumentError(absl::Substitute( + "field '$0' is not a message type", field_descriptor->name())); + } + return &(*message_type.value()); +} + +// Inserts the type name with an empty set into types_and_visible_fields. +// Returns an error if the type name is already present with a non-empty set. +absl::Status AddAllHiddenFields(TypeMap& types_and_visible_fields, + absl::string_view type_name) { + auto result = types_and_visible_fields.find(type_name); + if (result != types_and_visible_fields.end()) { + if (!result->second.empty()) { + return absl::InvalidArgumentError( + absl::Substitute("cannot insert a proto type mask with all hidden " + "fields when type '$0' has already been inserted " + "with a proto type mask with a visible field", + type_name)); + } + return absl::OkStatus(); + } + types_and_visible_fields.insert({std::string(type_name), {}}); + return absl::OkStatus(); +} + +// Inserts the type name and field name into types_and_visible_fields. +// Returns an error if the type name is already present with an empty set. +absl::Status AddVisibleField(TypeMap& types_and_visible_fields, + absl::string_view type_name, + absl::string_view field_name) { + auto result = types_and_visible_fields.find(type_name); + if (result != types_and_visible_fields.end()) { + if (result->second.empty()) { + return absl::InvalidArgumentError(absl::Substitute( + "cannot insert a proto type mask with visible " + "field '$0' when type '$1' has already been inserted " + "with a proto type mask with all hidden fields", + field_name, type_name)); + } + result->second.insert(std::string(field_name)); + return absl::OkStatus(); + } + types_and_visible_fields.insert( + {std::string(type_name), {std::string(field_name)}}); + return absl::OkStatus(); +} + +// Processes the input proto type masks to create and return the +// types_and_visible_fields map. +// Returns an error if one of the proto type masks is not valid. For example, +// if a type is not found in the descriptor pool, if a field name is not +// found, or if a field is not a message type when we are expecting it to be. +// Returns an error if there is a conflict in field visibility when +// updating the map. +absl::StatusOr ComputeVisibleFieldsMap( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + const std::vector& proto_type_masks) { + TypeMap types_and_visible_fields; + for (const ProtoTypeMask& proto_type_mask : proto_type_masks) { + absl::string_view type_name = proto_type_mask.GetTypeName(); + CEL_ASSIGN_OR_RETURN(const Descriptor* descriptor, + FindMessage(descriptor_pool, type_name)); + const absl::btree_set& field_paths = + proto_type_mask.GetFieldPaths(); + if (field_paths.empty()) { + CEL_RETURN_IF_ERROR( + AddAllHiddenFields(types_and_visible_fields, type_name)); + } + for (const FieldPath& field_path : field_paths) { + const Descriptor* target_descriptor = descriptor; + absl::Span field_selection = + field_path.GetFieldSelection(); + for (auto iterator = field_selection.begin(); + iterator != field_selection.end(); ++iterator) { + CEL_ASSIGN_OR_RETURN(const FieldDescriptor* field_descriptor, + FindField(target_descriptor, *iterator)); + CEL_RETURN_IF_ERROR(AddVisibleField(types_and_visible_fields, + target_descriptor->full_name(), + *iterator)); + if (std::next(iterator) != field_selection.end()) { + CEL_ASSIGN_OR_RETURN(target_descriptor, GetMessage(field_descriptor)); + } + } + } + } + return types_and_visible_fields; +} + +} // namespace + +absl::StatusOr> +ProtoTypeMaskRegistry::Create( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + const std::vector& proto_type_masks) { + CEL_ASSIGN_OR_RETURN( + auto types_and_visible_fields, + ComputeVisibleFieldsMap(descriptor_pool, proto_type_masks)); + std::shared_ptr proto_type_mask_registry = + absl::WrapUnique(new ProtoTypeMaskRegistry(types_and_visible_fields)); + return proto_type_mask_registry; +} + +bool ProtoTypeMaskRegistry::FieldIsVisible(absl::string_view type_name, + absl::string_view field_name) const { + auto iterator = types_and_visible_fields_.find(type_name); + if (iterator != types_and_visible_fields_.end() && + !iterator->second.contains(field_name)) { + return false; + } + return true; +} + +std::string ProtoTypeMaskRegistry::DebugString() const { + std::string output = "ProtoTypeMaskRegistry { "; + for (auto& element : types_and_visible_fields_) { + absl::StrAppend(&output, "{type: '", element.first, "', visible_fields: '", + absl::StrJoin(element.second, "', '"), "'} "); + } + absl::StrAppend(&output, "}"); + return output; +} + +} // namespace cel::checker_internal diff --git a/checker/internal/proto_type_mask_registry.h b/checker/internal/proto_type_mask_registry.h new file mode 100644 index 000000000..338353e7d --- /dev/null +++ b/checker/internal/proto_type_mask_registry.h @@ -0,0 +1,83 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_REGISTRY_H_ +#define THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_REGISTRY_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "checker/internal/proto_type_mask.h" +#include "google/protobuf/descriptor.h" + +namespace cel::checker_internal { + +// Stores information related to ProtoTypeMasks. Visibility is defined per type, +// meaning that all messages of a type have the same visible fields. +class ProtoTypeMaskRegistry { + public: + // Processes the input proto type masks to create a ProtoTypeMaskRegistry. + // Returns an error if one of the proto type masks is not valid. For example, + // if a type is not found in the descriptor pool, if a field name is not + // found, or if a field is not a message type when we are expecting it to be. + // Returns an error if there is a conflict in field visibility when + // updating the map. + static absl::StatusOr> Create( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + const std::vector& proto_type_masks); + + const absl::flat_hash_map>& + GetTypesAndVisibleFields() const { + return types_and_visible_fields_; + } + + // Returns true when the field name is visible. A field is visible if: + // 1. The type name is not a key in the map. + // 2. The type name is a key in the map and the field name is in the set of + // field names that are visible for the type. + bool FieldIsVisible(absl::string_view type_name, + absl::string_view field_name) const; + + template + friend void AbslStringify( + Sink& sink, + const std::shared_ptr& proto_type_mask_registry) { + sink.Append(proto_type_mask_registry->DebugString()); + } + + private: + explicit ProtoTypeMaskRegistry( + absl::flat_hash_map> + types_and_visible_fields) + : types_and_visible_fields_(std::move(types_and_visible_fields)) {} + + std::string DebugString() const; + + // Map of types that have a field mask where the keys are + // fully qualified type names and the values are the set of field names that + // are visible for the type. + absl::flat_hash_map> + types_and_visible_fields_; +}; + +} // namespace cel::checker_internal + +#endif // THIRD_PARTY_CEL_CPP_CHECKER_PROTO_TYPE_MASK_REGISTRY_H_ diff --git a/checker/internal/proto_type_mask_registry_test.cc b/checker/internal/proto_type_mask_registry_test.cc new file mode 100644 index 000000000..3a73c8823 --- /dev/null +++ b/checker/internal/proto_type_mask_registry_test.cc @@ -0,0 +1,402 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/proto_type_mask_registry.h" + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "checker/internal/proto_type_mask.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" + +namespace cel::checker_internal { +namespace { + +using ::absl_testing::StatusIs; +using ::cel::internal::GetSharedTestingDescriptorPool; +using ::testing::AllOf; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +using TypeMap = + absl::flat_hash_map>; + +TEST(ProtoTypeMaskRegistryTest, + CreateWithEmptyInputSucceedsAndAllFieldsAreVisible) { + std::vector proto_type_masks = {}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), IsEmpty()); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithEmptyTypeReturnsError) { + std::vector proto_type_masks = {ProtoTypeMask("", {})}; + EXPECT_THAT(ProtoTypeMaskRegistry::Create( + GetSharedTestingDescriptorPool().get(), proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type '' not found"))); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithUnknownTypeReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("com.example.UnknownType", {})}; + EXPECT_THAT(ProtoTypeMaskRegistry::Create( + GetSharedTestingDescriptorPool().get(), proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type 'com.example.UnknownType' not found"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithEmptySetFieldPathSucceedsAndFieldsAreHidden) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", IsEmpty()))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDuplicateEmptySetFieldPathSucceedsAndFieldsAreHidden) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {}), + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", IsEmpty()))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithEmptyFieldPathReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {""})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field '' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithDelimiterFieldPathReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", {"."})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field '' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithUnknownFieldReturnsError) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"unknown_field"})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field 'unknown_field' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthOneNonMessageFieldsSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"single_int32", "single_any", "single_timestamp"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("single_int32", "single_any", + "single_timestamp")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "single_int32")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "single_any")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "single_timestamp")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, CreateWithDepthTwoNonMessageFieldReturnsError) { + std::vector proto_type_masks; + proto_type_masks.push_back( + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"single_int32.any_field_name"})); + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("field 'single_int32' is not a message type"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthOneMessageFieldSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"standalone_message"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT( + proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre(Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("standalone_message")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "standalone_message")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthTwoMessageFieldSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"standalone_message.bb"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("standalone_message")), + Pair("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + UnorderedElementsAre("bb")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "standalone_message")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", "bb")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthTwoMessageUnknownFieldReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"standalone_message.unknown_field"})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr( + "could not select field 'unknown_field' from type " + "'cel.expr.conformance.proto3.TestAllTypes.NestedMessage'"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthThreeMessageFieldSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.NestedTestAllTypes", + UnorderedElementsAre("payload")), + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("standalone_message")), + Pair("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + UnorderedElementsAre("bb")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.NestedTestAllTypes", "payload")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.NestedTestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "standalone_message")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", "bb")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthOneRepeatedMessageFieldSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"repeated_nested_message"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT(proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("repeated_nested_message")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "repeated_nested_message")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithDepthTwoRepeatedMessageFieldReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"repeated_nested_message.bb"})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr("field 'repeated_nested_message' is not a message type"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateWithListOfFieldPathsSucceedsAndFieldsAreVisible) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb", "payload.single_int32"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT( + proto_type_mask_registry->GetTypesAndVisibleFields(), + UnorderedElementsAre( + Pair("cel.expr.conformance.proto3.NestedTestAllTypes", + UnorderedElementsAre("payload")), + Pair("cel.expr.conformance.proto3.TestAllTypes", + UnorderedElementsAre("standalone_message", "single_int32")), + Pair("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + UnorderedElementsAre("bb")))); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible("any_type_name", + "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.NestedTestAllTypes", "payload")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.NestedTestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "standalone_message")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "single_int32")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes", "any_field_name")); + EXPECT_TRUE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", "bb")); + EXPECT_FALSE(proto_type_mask_registry->FieldIsVisible( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + "any_field_name")); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateAddVisibleFieldThenAllHiddenFieldsReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"standalone_message.bb"}), + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + {})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr( + "cannot insert a proto type mask with all hidden fields when " + "type 'cel.expr.conformance.proto3.TestAllTypes.NestedMessage' " + "has already been inserted with a proto type mask with a visible " + "field"))); +} + +TEST(ProtoTypeMaskRegistryTest, + CreateAddAllHiddenThenVisibleFieldReturnsError) { + std::vector proto_type_masks = { + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes.NestedMessage", + {}), + ProtoTypeMask("cel.expr.conformance.proto3.TestAllTypes", + {"standalone_message.bb"})}; + EXPECT_THAT( + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr( + "cannot insert a proto type mask with visible field 'bb' when " + "type 'cel.expr.conformance.proto3.TestAllTypes.NestedMessage' " + "has already been inserted with a proto type mask with all " + "hidden fields"))); +} + +TEST(ProtoTypeMaskRegistryTest, AbslStringifyPrintsTypesAndVisibleFieldsMap) { + std::vector proto_type_masks = {ProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"standalone_message.bb"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr proto_type_mask_registry, + ProtoTypeMaskRegistry::Create(GetSharedTestingDescriptorPool().get(), + proto_type_masks)); + EXPECT_THAT( + absl::StrCat(proto_type_mask_registry), + AllOf(HasSubstr("ProtoTypeMaskRegistry {"), + HasSubstr("{type: 'cel.expr.conformance.proto3.TestAllTypes', " + "visible_fields: 'standalone_message'}"), + HasSubstr("{type: " + "'cel.expr.conformance.proto3.TestAllTypes.NestedMessage'" + ", visible_fields: 'bb'}"))); +} + +} // namespace +} // namespace cel::checker_internal diff --git a/checker/internal/proto_type_mask_test.cc b/checker/internal/proto_type_mask_test.cc new file mode 100644 index 000000000..0c534f8cf --- /dev/null +++ b/checker/internal/proto_type_mask_test.cc @@ -0,0 +1,143 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "checker/internal/proto_type_mask.h" + +#include +#include + +#include "absl/container/btree_set.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "checker/internal/field_path.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" + +namespace cel::checker_internal { +namespace { + +using ::absl_testing::StatusIs; +using ::cel::internal::GetSharedTestingDescriptorPool; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::UnorderedElementsAre; + +TEST(ProtoTypeMaskTest, EmptyTypeNameAndEmptyFieldPathsSucceeds) { + std::string type_name = ""; + std::vector field_paths; + ProtoTypeMask proto_type_mask(type_name, field_paths); + EXPECT_EQ(proto_type_mask.GetTypeName(), ""); + EXPECT_THAT(proto_type_mask.GetFieldPaths(), IsEmpty()); +} + +TEST(ProtoTypeMaskTest, NotEmptyTypeNameAndNotEmptyFieldPathsSucceeds) { + std::string type_name = "google.type.Expr"; + std::vector field_paths = {"resource.name", "resource.type"}; + ProtoTypeMask proto_type_mask(type_name, field_paths); + EXPECT_EQ(proto_type_mask.GetTypeName(), "google.type.Expr"); + EXPECT_THAT(proto_type_mask.GetFieldPaths(), + UnorderedElementsAre(FieldPath("resource.name"), + FieldPath("resource.type"))); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithEmptyTypeReturnsError) { + ProtoTypeMask proto_type_mask("", {}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type '' not found"))); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithUnknownTypeReturnsError) { + ProtoTypeMask proto_type_mask("com.example.UnknownType", {}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type 'com.example.UnknownType' not found"))); +} + +TEST(ProtoTypeMaskTest, + GetFieldNamesWithEmptySetFieldPathSucceedsAndReturnsEmptySet) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", {}); + ASSERT_OK_AND_ASSIGN( + absl::btree_set field_names, + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get())); + EXPECT_THAT(field_names, IsEmpty()); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithEmptyFieldPathReturnsError) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", + {""}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field '' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithDelimiterFieldPathReturnsError) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", + {"single_int32", "."}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field '' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskTest, GetFieldNamesWithUnknownFieldReturnsError) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", + {"unknown_field"}); + EXPECT_THAT( + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("could not select field 'unknown_field' from type " + "'cel.expr.conformance.proto3.TestAllTypes'"))); +} + +TEST(ProtoTypeMaskTest, + GetFieldNamesWithValidFieldsSucceedsAndReturnsFieldNames) { + ProtoTypeMask proto_type_mask("cel.expr.conformance.proto3.TestAllTypes", + {"single_int32", "single_string"}); + ASSERT_OK_AND_ASSIGN( + absl::btree_set field_names, + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get())); + EXPECT_THAT(field_names, + UnorderedElementsAre("single_int32", "single_string")); +} + +TEST(ProtoTypeMaskTest, + GetFieldNamesWithValidFieldPathsSucceedsAndReturnsFieldNames) { + ProtoTypeMask proto_type_mask( + "cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb", "payload.single_int32", + "child.any_field_name"}); + ASSERT_OK_AND_ASSIGN( + absl::btree_set field_names, + proto_type_mask.GetFieldNames(GetSharedTestingDescriptorPool().get())); + EXPECT_THAT(field_names, UnorderedElementsAre("payload", "child")); +} + +TEST(ProtoTypeMaskTest, AbslStringifyPrintsTypeNameAndFieldPaths) { + std::string type_name = "google.type.Expr"; + std::vector field_paths = {"resource.name", "resource.type"}; + ProtoTypeMask proto_type_mask(type_name, field_paths); + EXPECT_THAT(absl::StrCat(proto_type_mask), + HasSubstr("ProtoTypeMask { type name: 'google.type.Expr', field " + "paths: { 'resource.name', 'resource.type' } }")); +} + +} // namespace +} // namespace cel::checker_internal From dd4f36816db185068e3ec8ba3663c95eb3f930db Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 4 Jun 2026 15:58:40 -0700 Subject: [PATCH 051/350] Fix misuses of __cxa_demangle length parameters PiperOrigin-RevId: 926935330 --- common/typeinfo.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/common/typeinfo.cc b/common/typeinfo.cc index 86bae1934..b07275712 100644 --- a/common/typeinfo.cc +++ b/common/typeinfo.cc @@ -57,18 +57,13 @@ std::string TypeInfo::DebugString() const { } return std::string(demangled.get()); #else - size_t length = 0; int status = 0; std::unique_ptr demangled( - abi::__cxa_demangle(rep_->name(), nullptr, &length, &status)); + abi::__cxa_demangle(rep_->name(), nullptr, nullptr, &status)); if (status != 0 || demangled == nullptr) { return std::string(rep_->name()); } - while (length != 0 && demangled.get()[length - 1] == '\0') { - // length includes the null terminator, remove it. - --length; - } - return std::string(demangled.get(), length); + return std::string(demangled.get()); #endif #else return absl::StrCat("0x", absl::Hex(absl::bit_cast(rep_))); From 84e77731f38b4d7d0509311d9e05de6031014283 Mon Sep 17 00:00:00 2001 From: sahvx655-wq Date: Mon, 8 Jun 2026 20:59:27 +0530 Subject: [PATCH 052/350] fix out-of-range double-to-int UB in proto map key lookup --- common/values/parsed_map_field_value.cc | 48 +++++++------ common/values/parsed_map_field_value_test.cc | 72 ++++++++++++++++++++ 2 files changed, 100 insertions(+), 20 deletions(-) diff --git a/common/values/parsed_map_field_value.cc b/common/values/parsed_map_field_value.cc index 47b737f82..641b2f0e4 100644 --- a/common/values/parsed_map_field_value.cc +++ b/common/values/parsed_map_field_value.cc @@ -33,6 +33,7 @@ #include "extensions/protobuf/internal/map_reflection.h" #include "internal/json.h" #include "internal/message_equality.h" +#include "internal/number.h" #include "internal/status_macros.h" #include "internal/well_known_types.h" #include "google/protobuf/arena.h" @@ -197,11 +198,15 @@ absl::optional ValueAsInt32(const Value& value) { uint_value && uint_value->NativeValue() <= std::numeric_limits::max()) { return static_cast(uint_value->NativeValue()); - } else if (auto double_value = value.AsDouble(); - double_value && - static_cast(static_cast( - double_value->NativeValue())) == double_value->NativeValue()) { - return static_cast(double_value->NativeValue()); + } else if (auto double_value = value.AsDouble(); double_value) { + auto number = internal::Number::FromDouble(double_value->NativeValue()); + if (number.LosslessConvertibleToInt()) { + int64_t wide_value = number.AsInt(); + if (wide_value >= std::numeric_limits::min() && + wide_value <= std::numeric_limits::max()) { + return static_cast(wide_value); + } + } } return absl::nullopt; } @@ -213,11 +218,11 @@ absl::optional ValueAsInt64(const Value& value) { uint_value && uint_value->NativeValue() <= std::numeric_limits::max()) { return static_cast(uint_value->NativeValue()); - } else if (auto double_value = value.AsDouble(); - double_value && - static_cast(static_cast( - double_value->NativeValue())) == double_value->NativeValue()) { - return static_cast(double_value->NativeValue()); + } else if (auto double_value = value.AsDouble(); double_value) { + auto number = internal::Number::FromDouble(double_value->NativeValue()); + if (number.LosslessConvertibleToInt()) { + return number.AsInt(); + } } return absl::nullopt; } @@ -231,11 +236,14 @@ absl::optional ValueAsUInt32(const Value& value) { uint_value && uint_value->NativeValue() <= std::numeric_limits::max()) { return static_cast(uint_value->NativeValue()); - } else if (auto double_value = value.AsDouble(); - double_value && - static_cast(static_cast( - double_value->NativeValue())) == double_value->NativeValue()) { - return static_cast(double_value->NativeValue()); + } else if (auto double_value = value.AsDouble(); double_value) { + auto number = internal::Number::FromDouble(double_value->NativeValue()); + if (number.LosslessConvertibleToUint()) { + uint64_t wide_value = number.AsUint(); + if (wide_value <= std::numeric_limits::max()) { + return static_cast(wide_value); + } + } } return absl::nullopt; } @@ -246,11 +254,11 @@ absl::optional ValueAsUInt64(const Value& value) { return static_cast(int_value->NativeValue()); } else if (auto uint_value = value.AsUint(); uint_value) { return uint_value->NativeValue(); - } else if (auto double_value = value.AsDouble(); - double_value && - static_cast(static_cast( - double_value->NativeValue())) == double_value->NativeValue()) { - return static_cast(double_value->NativeValue()); + } else if (auto double_value = value.AsDouble(); double_value) { + auto number = internal::Number::FromDouble(double_value->NativeValue()); + if (number.LosslessConvertibleToUint()) { + return number.AsUint(); + } } return absl::nullopt; } diff --git a/common/values/parsed_map_field_value_test.cc b/common/values/parsed_map_field_value_test.cc index 271813f40..491aab2e1 100644 --- a/common/values/parsed_map_field_value_test.cc +++ b/common/values/parsed_map_field_value_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include @@ -225,6 +226,77 @@ TEST_F(ParsedMapFieldValueTest, Find) { IsOkAndHolds(Eq(absl::nullopt))); } +TEST_F(ParsedMapFieldValueTest, FindDoubleKeyConversion) { + // Lookups on integer-keyed maps coerce double keys following CEL's numeric + // semantics ({1: 'x'}[1.0] resolves to key 1). A double that is out of range, + // negative for an unsigned key, or non-finite must be reported as absent + // rather than narrowed to the key type: converting such a double straight to + // an integer with static_cast is undefined behaviour. + ParsedMapFieldValue int64_map( + DynamicParseTextProto(R"pb( + map_int64_int64 { key: 1 value: 100 } + )pb"), + DynamicGetField("map_int64_int64"), arena()); + EXPECT_THAT(int64_map.Find(DoubleValue(1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(100)))); + EXPECT_THAT(int64_map.Find(DoubleValue(1e19), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(int64_map.Find(DoubleValue(-1e19), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(int64_map.Find(DoubleValue(std::numeric_limits::infinity()), + descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT( + int64_map.Find(DoubleValue(std::numeric_limits::quiet_NaN()), + descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); + + ParsedMapFieldValue uint64_map( + DynamicParseTextProto(R"pb( + map_uint64_uint64 { key: 1 value: 100 } + )pb"), + DynamicGetField("map_uint64_uint64"), arena()); + EXPECT_THAT(uint64_map.Find(DoubleValue(1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(UintValueIs(100)))); + EXPECT_THAT(uint64_map.Find(DoubleValue(-1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(uint64_map.Find(DoubleValue(1e20), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); + + ParsedMapFieldValue int32_map( + DynamicParseTextProto(R"pb( + map_int32_int32 { key: 1 value: 100 } + )pb"), + DynamicGetField("map_int32_int32"), arena()); + EXPECT_THAT(int32_map.Find(DoubleValue(1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(100)))); + EXPECT_THAT(int32_map.Find(DoubleValue(1e19), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); + + ParsedMapFieldValue uint32_map( + DynamicParseTextProto(R"pb( + map_uint32_uint32 { key: 1 value: 100 } + )pb"), + DynamicGetField("map_uint32_uint32"), arena()); + EXPECT_THAT(uint32_map.Find(DoubleValue(1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(UintValueIs(100)))); + EXPECT_THAT(uint32_map.Find(DoubleValue(-1.0), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(uint32_map.Find(DoubleValue(1e19), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(absl::nullopt))); +} + TEST_F(ParsedMapFieldValueTest, Has) { ParsedMapFieldValue value( DynamicParseTextProto(R"pb( From 35957777f1d8c22f3c4ac5e9cc333c81f559fc8c Mon Sep 17 00:00:00 2001 From: Tristan Swadell Date: Mon, 8 Jun 2026 14:25:17 -0700 Subject: [PATCH 053/350] Variadic logical operators PiperOrigin-RevId: 928770216 --- checker/internal/BUILD | 2 + checker/internal/type_checker_impl.cc | 7 +- checker/internal/type_checker_impl_test.cc | 88 ++++++ checker/internal/type_inference_context.cc | 16 +- eval/compiler/flat_expr_builder.cc | 203 +++++++------ eval/compiler/flat_expr_builder_test.cc | 319 ++++++++++++++++++--- parser/options.h | 4 + parser/parser.cc | 21 +- parser/parser_test.cc | 53 ++++ tools/cel_unparser.cc | 25 +- tools/cel_unparser_test.cc | 19 ++ 11 files changed, 613 insertions(+), 144 deletions(-) diff --git a/checker/internal/BUILD b/checker/internal/BUILD index 25550616a..26c7b543f 100644 --- a/checker/internal/BUILD +++ b/checker/internal/BUILD @@ -145,6 +145,7 @@ cc_library( "//common:container", "//common:decl", "//common:expr", + "//common:standard_definitions", "//common:type", "//common:type_kind", "//internal:lexis", @@ -238,6 +239,7 @@ cc_library( deps = [ ":format_type_name", "//common:decl", + "//common:standard_definitions", "//common:type", "//common:type_kind", "@com_google_absl//absl/container:flat_hash_map", diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 1ce871255..6b6b051b1 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -52,6 +52,7 @@ #include "common/constant.h" #include "common/decl.h" #include "common/expr.h" +#include "common/standard_definitions.h" #include "common/type.h" #include "common/type_kind.h" #include "internal/status_macros.h" @@ -894,8 +895,12 @@ const FunctionDecl* ResolveVisitor::ResolveFunctionCallShape( if (decl == nullptr) { return true; } + bool is_logical_op = (candidate == cel::StandardFunctions::kAnd || + candidate == cel::StandardFunctions::kOr) && + arg_count >= 2; for (const auto& ovl : decl->overloads()) { - if (ovl.member() == is_receiver && ovl.args().size() == arg_count) { + if (ovl.member() == is_receiver && + (ovl.args().size() == arg_count || is_logical_op)) { return false; } } diff --git a/checker/internal/type_checker_impl_test.cc b/checker/internal/type_checker_impl_test.cc index 893f0689d..61ef7d55b 100644 --- a/checker/internal/type_checker_impl_test.cc +++ b/checker/internal/type_checker_impl_test.cc @@ -55,6 +55,7 @@ #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" namespace cel { namespace checker_internal { @@ -1471,6 +1472,93 @@ TEST(TypeCheckerImplTest, TypeInferredFromStructCreation) { std::make_unique(DynTypeSpec()))))))); } +struct VariadicLogicalCheckerTestCase { + std::string expr; +}; + +class VariadicLogicalCheckerTest + : public testing::TestWithParam {}; + +TEST_P(VariadicLogicalCheckerTest, Check) { + const auto& test_case = GetParam(); + + auto builder = cel::NewParserBuilder(); + builder->GetOptions().enable_variadic_logical_operators = true; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.expr)); + ASSERT_OK_AND_ASSIGN(auto parsed_ast, parser->Parse(*source)); + + google::protobuf::Arena arena; + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + TypeCheckerImpl impl(std::move(env)); + auto checker_builder = impl.ToBuilder(); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("a", BoolType())), + IsOk()); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("b", BoolType())), + IsOk()); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("c", BoolType())), + IsOk()); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("d", BoolType())), + IsOk()); + ASSERT_THAT(checker_builder->AddVariable(MakeVariableDecl("e", BoolType())), + IsOk()); + + ASSERT_OK_AND_ASSIGN(auto checker, checker_builder->Build()); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + checker->Check(std::move(parsed_ast))); + + ASSERT_TRUE(result.IsValid()) + << absl::StrJoin(result.GetIssues(), "\n", + [](std::string* out, const TypeCheckIssue& issue) { + absl::StrAppend(out, issue.message()); + }); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr checked_ast, result.ReleaseAst()); + EXPECT_THAT(checked_ast->type_map(), + Contains(Pair(checked_ast->root_expr().id(), + Eq(AstType(PrimitiveType::kBool))))); +} + +INSTANTIATE_TEST_SUITE_P( + VariadicLogicalChecker, VariadicLogicalCheckerTest, + testing::Values(VariadicLogicalCheckerTestCase{"true && false && true"}, + VariadicLogicalCheckerTestCase{"a && b && c && d"}, + VariadicLogicalCheckerTestCase{"a || b || c || d"}, + VariadicLogicalCheckerTestCase{"a && b && (c || d || e)"}, + VariadicLogicalCheckerTestCase{"a && b && c"}, + VariadicLogicalCheckerTestCase{"a || b || c"}, + VariadicLogicalCheckerTestCase{"[a, b, c].exists(x, x)"}, + VariadicLogicalCheckerTestCase{"[a, b, c].all(x, x)"})); + +TEST(TypeCheckerImplTest, VariadicLogicalOperatorsError) { + cel::expr::ParsedExpr parsed_expr; + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr { + call_expr { + function: "_&&_" + args { const_expr { bool_value: true } } + } + } + )pb", + &parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto parsed_ast, + cel::CreateAstFromParsedExpr(parsed_expr)); + + google::protobuf::Arena arena; + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + impl.Check(std::move(parsed_ast))); + + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.GetIssues(), + Contains(IsIssueWithSubstring(Severity::kError, "undeclared reference"))); +} + TEST(TypeCheckerImplTest, ExpectedTypeMatches) { google::protobuf::Arena arena; TypeCheckEnv env(GetSharedTestingDescriptorPool()); diff --git a/checker/internal/type_inference_context.cc b/checker/internal/type_inference_context.cc index 5b909d982..1a87d9e15 100644 --- a/checker/internal/type_inference_context.cc +++ b/checker/internal/type_inference_context.cc @@ -30,6 +30,7 @@ #include "absl/types/span.h" #include "checker/internal/format_type_name.h" #include "common/decl.h" +#include "common/standard_definitions.h" #include "common/type.h" #include "common/type_kind.h" @@ -537,21 +538,28 @@ TypeInferenceContext::ResolveOverload(const FunctionDecl& decl, bool is_receiver) { std::optional result_type; + bool is_logical_op = (decl.name() == cel::StandardFunctions::kAnd || + decl.name() == cel::StandardFunctions::kOr) && + argument_types.size() >= 2; + std::vector matching_overloads; for (const auto& ovl : decl.overloads()) { if (ovl.member() != is_receiver || - argument_types.size() != ovl.args().size()) { + (!is_logical_op && argument_types.size() != ovl.args().size())) { continue; } auto call_type_instance = InstantiateFunctionOverload(*this, ovl); - ABSL_DCHECK_EQ(argument_types.size(), - call_type_instance.param_types.size()); + if (!is_logical_op) { + ABSL_DCHECK_EQ(argument_types.size(), + call_type_instance.param_types.size()); + } bool is_match = true; AssignabilityContext assignability_context = CreateAssignabilityContext(); for (int i = 0; i < argument_types.size(); ++i) { + int param_index = is_logical_op ? 0 : i; if (!assignability_context.IsAssignable( - argument_types[i], call_type_instance.param_types[i])) { + argument_types[i], call_type_instance.param_types[param_index])) { is_match = false; break; } diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 1e3f4ecd3..d6ccdf040 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -232,7 +232,7 @@ class BinaryCondVisitor : public CondVisitor { private: FlatExprVisitor* visitor_; const BinaryCond cond_; - Jump jump_step_; + std::vector jump_steps_; bool short_circuiting_; }; @@ -622,7 +622,7 @@ class FlatExprVisitor : public cel::AstVisitor { program_optimizers_) { absl::Status status = optimizer->OnPreVisit(extension_context_, expr); if (!status.ok()) { - SetProgressStatusError(status); + SetProgressStatusIfError(status); } } } @@ -639,7 +639,7 @@ class FlatExprVisitor : public cel::AstVisitor { program_optimizers_) { absl::Status status = optimizer->OnPostVisit(extension_context_, expr); if (!status.ok()) { - SetProgressStatusError(status); + SetProgressStatusIfError(status); return; } } @@ -657,7 +657,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (!comprehension_stack_.empty() && comprehension_stack_.back().is_optimizable_bind && (&comprehension_stack_.back().comprehension->accu_init() == &expr)) { - SetProgressStatusError( + SetProgressStatusIfError( MaybeExtractSubexpression(&expr, comprehension_stack_.back())); } @@ -666,7 +666,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (block.current_binding == &expr) { int index = program_builder_.ExtractSubexpression(&expr); if (index == -1) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("failed to extract subexpression")); return; } @@ -686,7 +686,7 @@ class FlatExprVisitor : public cel::AstVisitor { ConvertConstant(const_expr, cel::NewDeleteAllocator()); if (!converted_value.ok()) { - SetProgressStatusError(converted_value.status()); + SetProgressStatusIfError(converted_value.status()); return; } @@ -722,13 +722,13 @@ class FlatExprVisitor : public cel::AstVisitor { if (absl::ConsumePrefix(&index_suffix, "@index")) { size_t index; if (!absl::SimpleAtoi(index_suffix, &index)) { - SetProgressStatusError( + SetProgressStatusIfError( issue_collector_.AddIssue(RuntimeIssue::CreateError( absl::InvalidArgumentError("bad @index")))); return {-1, -1}; } if (index >= block.size) { - SetProgressStatusError( + SetProgressStatusIfError( issue_collector_.AddIssue(RuntimeIssue::CreateError( absl::InvalidArgumentError(absl::StrCat( "invalid @index greater than number of bindings: ", @@ -736,7 +736,7 @@ class FlatExprVisitor : public cel::AstVisitor { return {-1, -1}; } if (index >= block.current_index) { - SetProgressStatusError( + SetProgressStatusIfError( issue_collector_.AddIssue(RuntimeIssue::CreateError( absl::InvalidArgumentError(absl::StrCat( "@index references current or future binding: ", index, @@ -754,7 +754,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (record.iter_var_in_scope && record.comprehension->iter_var() == path) { if (record.is_optimizable_bind) { - SetProgressStatusError(issue_collector_.AddIssue( + SetProgressStatusIfError(issue_collector_.AddIssue( RuntimeIssue::CreateWarning(absl::InvalidArgumentError( "Unexpected iter_var access in trivial comprehension")))); return {-1, -1}; @@ -781,7 +781,7 @@ class FlatExprVisitor : public cel::AstVisitor { // If we see a CSE generated comprehension variable that was not // resolvable through the normal comprehension scope resolution, reject it // now rather than surfacing errors at activation time. - SetProgressStatusError( + SetProgressStatusIfError( issue_collector_.AddIssue(RuntimeIssue::CreateError( absl::InvalidArgumentError("out of scope reference to CSE " "generated comprehension variable")))); @@ -811,7 +811,7 @@ class FlatExprVisitor : public cel::AstVisitor { auto* subexpression = program_builder_.GetExtractedSubexpression(slot.subexpression); if (subexpression == nullptr) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InternalError("bad subexpression reference")); return; } @@ -965,7 +965,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (auto depth = RecursionEligible(); depth.has_value()) { auto deps = ExtractRecursiveDependencies(); if (deps.size() != 1) { - SetProgressStatusError(absl::InternalError( + SetProgressStatusIfError(absl::InternalError( "unexpected number of dependencies for select operation.")); return; } @@ -1022,7 +1022,7 @@ class FlatExprVisitor : public cel::AstVisitor { // cel.@block if (block_.has_value()) { // There can only be one for now. - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("multiple cel.@block are not allowed")); return; } @@ -1030,17 +1030,17 @@ class FlatExprVisitor : public cel::AstVisitor { BlockInfo& block = *block_; block.in = true; if (call_expr.args().empty()) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "malformed cel.@block: missing list of bound expressions")); return; } if (call_expr.args().size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "malformed cel.@block: missing bound expression")); return; } if (!call_expr.args()[0].has_list_expr()) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("malformed cel.@block: first argument " "is not a list of bound expressions")); return; @@ -1051,7 +1051,7 @@ class FlatExprVisitor : public cel::AstVisitor { block.bindings_set.reserve(block.size); for (const auto& list_expr_element : list_expr.elements()) { if (list_expr_element.optional()) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("malformed cel.@block: list of bound " "expressions contains an optional")); return; @@ -1093,7 +1093,7 @@ class FlatExprVisitor : public cel::AstVisitor { void MakeTernaryRecursive(const cel::Expr* expr) { if (expr->call_expr().args().size() != 3) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "unexpected number of args for builtin ternary")); return; } @@ -1109,7 +1109,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (condition_plan == nullptr || !condition_plan->IsRecursive() || left_plan == nullptr || !left_plan->IsRecursive() || right_plan == nullptr || !right_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); + SetProgressStatusIfError(FailedRecursivePlanning()); return; } @@ -1126,45 +1126,52 @@ class FlatExprVisitor : public cel::AstVisitor { } void MakeShortcircuitRecursive(const cel::Expr* expr, bool is_or) { - if (expr->call_expr().args().size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( + int args_size = expr->call_expr().args().size(); + if (args_size < 2) { + SetProgressStatusIfError(absl::InvalidArgumentError( "unexpected number of args for builtin boolean operator &&/||")); return; } - const cel::Expr* left_expr = &expr->call_expr().args()[0]; - const cel::Expr* right_expr = &expr->call_expr().args()[1]; - auto* left_plan = program_builder_.GetSubexpression(left_expr); - auto* right_plan = program_builder_.GetSubexpression(right_expr); - - if (left_plan == nullptr || !left_plan->IsRecursive() || - right_plan == nullptr || !right_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); + auto* current_plan = + program_builder_.GetSubexpression(&expr->call_expr().args()[0]); + if (current_plan == nullptr || !current_plan->IsRecursive()) { + SetProgressStatusIfError(FailedRecursivePlanning()); return; } + int current_depth = current_plan->recursive_program().depth; + std::unique_ptr current_step = + current_plan->ExtractRecursiveProgram().step; - int max_depth = std::max({0, left_plan->recursive_program().depth, - right_plan->recursive_program().depth}); - - if (is_or) { - SetRecursiveStep( - CreateDirectOrStep(left_plan->ExtractRecursiveProgram().step, - right_plan->ExtractRecursiveProgram().step, - expr->id(), options_.short_circuiting), - max_depth + 1); - } else { - SetRecursiveStep( - CreateDirectAndStep(left_plan->ExtractRecursiveProgram().step, - right_plan->ExtractRecursiveProgram().step, - expr->id(), options_.short_circuiting), - max_depth + 1); + for (int i = 1; i < args_size; ++i) { + auto* next_plan = + program_builder_.GetSubexpression(&expr->call_expr().args()[i]); + if (next_plan == nullptr || !next_plan->IsRecursive()) { + SetProgressStatusIfError(FailedRecursivePlanning()); + return; + } + current_depth = + std::max(current_depth, next_plan->recursive_program().depth); + std::unique_ptr next_step = + next_plan->ExtractRecursiveProgram().step; + if (is_or) { + current_step = + CreateDirectOrStep(std::move(current_step), std::move(next_step), + expr->id(), options_.short_circuiting); + } else { + current_step = + CreateDirectAndStep(std::move(current_step), std::move(next_step), + expr->id(), options_.short_circuiting); + } + current_depth++; } + SetRecursiveStep(std::move(current_step), current_depth); } void MakeOptionalShortcircuit(const cel::Expr* expr, bool is_or_value) { if (!expr->call_expr().has_target() || expr->call_expr().args().size() != 1) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "unexpected number of args for optional.or{Value}")); return; } @@ -1176,7 +1183,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (left_plan == nullptr || !left_plan->IsRecursive() || right_plan == nullptr || !right_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); + SetProgressStatusIfError(FailedRecursivePlanning()); return; } int max_depth = std::max({0, left_plan->recursive_program().depth, @@ -1200,7 +1207,7 @@ class FlatExprVisitor : public cel::AstVisitor { program_builder_.GetSubexpression(&comprehension->result()); if (result_plan == nullptr || !result_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); + SetProgressStatusIfError(FailedRecursivePlanning()); return; } @@ -1234,7 +1241,7 @@ class FlatExprVisitor : public cel::AstVisitor { loop_plan == nullptr || !loop_plan->IsRecursive() || condition_plan == nullptr || !condition_plan->IsRecursive() || result_plan == nullptr || !result_plan->IsRecursive()) { - SetProgressStatusError(FailedRecursivePlanning()); + SetProgressStatusIfError(FailedRecursivePlanning()); return; } @@ -1462,7 +1469,7 @@ class FlatExprVisitor : public cel::AstVisitor { return; } - SetProgressStatusError(comprehension_stack_.back().visitor->PostVisitArg( + SetProgressStatusIfError(comprehension_stack_.back().visitor->PostVisitArg( comprehension_arg, comprehension_stack_.back().expr)); } @@ -1524,7 +1531,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (std::optional depth = RecursionEligible(); depth.has_value()) { auto deps = ExtractRecursiveDependencies(); if (deps.size() != list_expr.elements().size()) { - SetProgressStatusError(absl::InternalError( + SetProgressStatusIfError(absl::InternalError( "Unexpected number of plan elements for CreateList expr")); return; } @@ -1547,7 +1554,7 @@ class FlatExprVisitor : public cel::AstVisitor { auto status_or_resolved_fields = ResolveCreateStructFields(struct_expr, expr.id()); if (!status_or_resolved_fields.ok()) { - SetProgressStatusError(status_or_resolved_fields.status()); + SetProgressStatusIfError(status_or_resolved_fields.status()); return; } std::string resolved_name = @@ -1558,7 +1565,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (auto depth = RecursionEligible(); depth.has_value()) { auto deps = ExtractRecursiveDependencies(); if (deps.size() != struct_expr.fields().size()) { - SetProgressStatusError(absl::InternalError( + SetProgressStatusIfError(absl::InternalError( "Unexpected number of plan elements for CreateStruct expr")); return; } @@ -1599,7 +1606,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (auto depth = RecursionEligible(); depth.has_value()) { auto deps = ExtractRecursiveDependencies(); if (deps.size() != 2 * map_expr.entries().size()) { - SetProgressStatusError(absl::InternalError( + SetProgressStatusIfError(absl::InternalError( "Unexpected number of plan elements for CreateStruct expr")); return; } @@ -1661,7 +1668,7 @@ class FlatExprVisitor : public cel::AstVisitor { "No overloads provided for FunctionStep creation"), RuntimeIssue::ErrorCode::kNoMatchingOverload)); if (!status.ok()) { - SetProgressStatusError(status); + SetProgressStatusIfError(status); return; } } @@ -1692,7 +1699,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (step.ok()) { return AddStep(*std::move(step)); } else { - SetProgressStatusError(step.status()); + SetProgressStatusIfError(step.status()); } return nullptr; } @@ -1711,19 +1718,19 @@ class FlatExprVisitor : public cel::AstVisitor { return; } if (program_builder_.current() == nullptr) { - SetProgressStatusError(absl::InternalError( + SetProgressStatusIfError(absl::InternalError( "CEL AST traversal out of order in flat_expr_builder.")); return; } program_builder_.current()->set_recursive_program(std::move(step), depth); if (depth > max_recursion_depth_) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( absl::StrCat("Maximum recursion depth of ", options_.max_recursion_depth, " exceeded"))); } } - void SetProgressStatusError(const absl::Status& status) { + void SetProgressStatusIfError(const absl::Status& status) { if (progress_status_.ok() && !status.ok()) { progress_status_ = status; } @@ -1765,7 +1772,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (valid_expression) { return true; } - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( absl::StrCat(error_message, message_parts...))); return false; } @@ -1947,7 +1954,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleIndex( if (auto depth = RecursionEligible(); depth.has_value()) { auto args = ExtractRecursiveDependencies(); if (args.size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "unexpected number of args for builtin index operator")); return CallHandlerResult::kIntercepted; } @@ -1974,7 +1981,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNot( if (auto depth = RecursionEligible(); depth.has_value()) { auto args = ExtractRecursiveDependencies(); if (args.size() != 1) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "unexpected number of args for builtin not operator")); return CallHandlerResult::kIntercepted; } @@ -1997,7 +2004,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNotStrictlyFalse( if (auto depth = RecursionEligible(); depth.has_value()) { auto args = ExtractRecursiveDependencies(); if (args.size() != 1) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("unexpected number of args for builtin " "@not_strictly_false operator")); return CallHandlerResult::kIntercepted; @@ -2016,7 +2023,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleBlock( ABSL_DCHECK(call_expr.function() == kBlock); if (!block_.has_value() || block_->expr != &expr || call_expr.args().size() != 2 || call_expr.has_target()) { - SetProgressStatusError( + SetProgressStatusIfError( absl::InvalidArgumentError("unexpected call to internal cel.@block")); return CallHandlerResult::kIntercepted; } @@ -2101,7 +2108,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleHeterogeneousEquality( if (auto depth = RecursionEligible(); depth.has_value()) { auto args = ExtractRecursiveDependencies(); if (args.size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "unexpected number of args for builtin equality operator")); return CallHandlerResult::kIntercepted; } @@ -2126,7 +2133,7 @@ FlatExprVisitor::HandleHeterogeneousEqualityIn(const cel::Expr& expr, if (auto depth = RecursionEligible(); depth.has_value()) { auto args = ExtractRecursiveDependencies(); if (args.size() != 2) { - SetProgressStatusError(absl::InvalidArgumentError( + SetProgressStatusIfError(absl::InvalidArgumentError( "unexpected number of args for builtin 'in' operator")); return CallHandlerResult::kIntercepted; } @@ -2164,13 +2171,14 @@ void BinaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { if (visitor_->PlanRecursiveProgram()) { return; } - if (short_circuiting_ && arg_num == 0 && + const int last_arg_index = expr->call_expr().args().size() - 1; + if (short_circuiting_ && arg_num < last_arg_index && (cond_ == BinaryCond::kAnd || cond_ == BinaryCond::kOr)) { // If first branch evaluation result is enough to determine output, // jump over the second branch and provide result of the first argument as // final output. - // Retain a pointer to the jump step so we can update the target after - // planning the second argument. + // Retain pointers to the jump steps so we can update the target after + // planning the next arguments. std::unique_ptr jump_step; switch (cond_) { case BinaryCond::kAnd: @@ -2185,7 +2193,7 @@ void BinaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { ProgramStepIndex index = visitor_->GetCurrentIndex(); if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); jump_step_ptr) { - jump_step_ = Jump(index, jump_step_ptr); + jump_steps_.push_back(Jump(index, jump_step_ptr)); } } } @@ -2215,7 +2223,7 @@ void BinaryCondVisitor::PostVisitTarget(const cel::Expr* expr) { ProgramStepIndex index = visitor_->GetCurrentIndex(); if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); jump_step_ptr) { - jump_step_ = Jump(index, jump_step_ptr); + jump_steps_.push_back(Jump(index, jump_step_ptr)); } } } @@ -2243,28 +2251,36 @@ void BinaryCondVisitor::PostVisit(const cel::Expr* expr) { return; } - switch (cond_) { - case BinaryCond::kAnd: - visitor_->AddStep(CreateAndStep(expr->id())); - break; - case BinaryCond::kOr: - visitor_->AddStep(CreateOrStep(expr->id())); - break; - case BinaryCond::kOptionalOr: - visitor_->AddStep( - CreateOptionalOrStep(/*is_or_value=*/false, expr->id())); - break; - case BinaryCond::kOptionalOrValue: - visitor_->AddStep(CreateOptionalOrStep(/*is_or_value=*/true, expr->id())); - break; - default: - ABSL_UNREACHABLE(); + int args_count = (cond_ == BinaryCond::kAnd || cond_ == BinaryCond::kOr) + ? expr->call_expr().args().size() + : 2; + for (int i = 0; i < args_count - 1; ++i) { + switch (cond_) { + case BinaryCond::kAnd: + visitor_->AddStep(CreateAndStep(expr->id())); + break; + case BinaryCond::kOr: + visitor_->AddStep(CreateOrStep(expr->id())); + break; + case BinaryCond::kOptionalOr: + visitor_->AddStep( + CreateOptionalOrStep(/*is_or_value=*/false, expr->id())); + break; + case BinaryCond::kOptionalOrValue: + visitor_->AddStep( + CreateOptionalOrStep(/*is_or_value=*/true, expr->id())); + break; + default: + ABSL_UNREACHABLE(); + } } if (short_circuiting_) { // If short-circuiting is enabled, point the conditional jump past the // boolean operator step. - visitor_->SetProgressStatusError( - jump_step_.set_target(visitor_->GetCurrentIndex())); + for (auto& jump : jump_steps_) { + visitor_->SetProgressStatusIfError( + jump.set_target(visitor_->GetCurrentIndex())); + } } } @@ -2321,7 +2337,7 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { if (visitor_->ValidateOrError( jump_to_second_.exists(), "Error configuring ternary operator: jump_to_second_ is null")) { - visitor_->SetProgressStatusError( + visitor_->SetProgressStatusIfError( jump_to_second_.set_target(visitor_->GetCurrentIndex())); } } @@ -2339,13 +2355,13 @@ void TernaryCondVisitor::PostVisit(const cel::Expr* expr) { if (visitor_->ValidateOrError( error_jump_.exists(), "Error configuring ternary operator: error_jump_ is null")) { - visitor_->SetProgressStatusError( + visitor_->SetProgressStatusIfError( error_jump_.set_target(visitor_->GetCurrentIndex())); } if (visitor_->ValidateOrError( jump_after_first_.exists(), "Error configuring ternary operator: jump_after_first_ is null")) { - visitor_->SetProgressStatusError( + visitor_->SetProgressStatusIfError( jump_after_first_.set_target(visitor_->GetCurrentIndex())); } } @@ -2403,7 +2419,8 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( break; } Jump jump_helper(index, jump_to_next); - visitor_->SetProgressStatusError(jump_helper.set_target(next_step_pos_)); + visitor_->SetProgressStatusIfError( + jump_helper.set_target(next_step_pos_)); // Set offsets jumping to the result step. if (cond_step_) { diff --git a/eval/compiler/flat_expr_builder_test.cc b/eval/compiler/flat_expr_builder_test.cc index d84007485..e2581e3fd 100644 --- a/eval/compiler/flat_expr_builder_test.cc +++ b/eval/compiler/flat_expr_builder_test.cc @@ -469,7 +469,7 @@ TEST(FlatExprBuilderTest, IdentExprUnsetName) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"(ident_expr {})", &expr); + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"(ident_expr {})", &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -482,10 +482,10 @@ TEST(FlatExprBuilderTest, SelectExprUnsetField) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"(select_expr{ + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"(select_expr{ operand{ ident_expr {name: 'var'} } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -498,11 +498,11 @@ TEST(FlatExprBuilderTest, SelectExprUnsetOperand) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"(select_expr{ + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"(select_expr{ field: 'field' operand { id: 1 } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -515,7 +515,8 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetAccuVar) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"(comprehension_expr{})", &expr); + ASSERT_TRUE( + google::protobuf::TextFormat::ParseFromString(R"(comprehension_expr{})", &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -527,10 +528,10 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetIterVar) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{accu_var: "a"} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -542,12 +543,12 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetAccuInit) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{ accu_var: "a" iter_var: "b"} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -559,7 +560,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetLoopCondition) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{ accu_var: 'a' iter_var: 'b' @@ -567,7 +568,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetLoopCondition) { const_expr {bool_value: true} }} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -579,7 +580,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetLoopStep) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{ accu_var: 'a' iter_var: 'b' @@ -590,7 +591,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetLoopStep) { const_expr {bool_value: true} }} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -602,7 +603,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetResult) { Expr expr; SourceInfo source_info; // An empty ident without the name set should error. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr{ accu_var: 'a' iter_var: 'b' @@ -616,7 +617,7 @@ TEST(FlatExprBuilderTest, ComprehensionExprUnsetResult) { const_expr {bool_value: false} }} )", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); EXPECT_THAT(builder.CreateExpression(&expr, &source_info).status(), @@ -628,7 +629,7 @@ TEST(FlatExprBuilderTest, MapComprehension) { Expr expr; SourceInfo source_info; // {1: "", 2: ""}.all(x, x > 0) - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( comprehension_expr { iter_var: "k" accu_var: "accu" @@ -665,7 +666,7 @@ TEST(FlatExprBuilderTest, MapComprehension) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -683,7 +684,7 @@ TEST(FlatExprBuilderTest, InvalidContainer) { Expr expr; SourceInfo source_info; // foo && bar - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( call_expr { function: "_&&_" args { @@ -697,7 +698,7 @@ TEST(FlatExprBuilderTest, InvalidContainer) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -909,7 +910,7 @@ TEST(FlatExprBuilderTest, ParsedNamespacedFunctionSupportDisabled) { TEST(FlatExprBuilderTest, BasicCheckedExprSupport) { CheckedExpr expr; // foo && bar - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( expr { id: 1 call_expr { @@ -928,7 +929,7 @@ TEST(FlatExprBuilderTest, BasicCheckedExprSupport) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -946,7 +947,7 @@ TEST(FlatExprBuilderTest, BasicCheckedExprSupport) { TEST(FlatExprBuilderTest, CheckedExprWithReferenceMap) { CheckedExpr expr; // `foo.var1` && `bar.var2` - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( reference_map { key: 2 value { @@ -988,7 +989,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMap) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); builder.flat_expr_builder().AddAstTransform( @@ -1008,7 +1009,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMap) { TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapFunction) { CheckedExpr expr; // ext.and(var1, bar.var2) - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( reference_map { key: 1 value { @@ -1057,7 +1058,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapFunction) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); builder.flat_expr_builder().AddAstTransform( @@ -1082,7 +1083,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapFunction) { TEST(FlatExprBuilderTest, CheckedExprActivationMissesReferences) { CheckedExpr expr; // && . - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( reference_map { key: 2 value { @@ -1125,7 +1126,7 @@ TEST(FlatExprBuilderTest, CheckedExprActivationMissesReferences) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); builder.flat_expr_builder().AddAstTransform( @@ -1160,7 +1161,7 @@ TEST(FlatExprBuilderTest, CheckedExprActivationMissesReferences) { TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapAndConstantFolding) { CheckedExpr expr; // {`var1`: 'hello'} - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( reference_map { key: 3 value { @@ -1190,7 +1191,7 @@ TEST(FlatExprBuilderTest, CheckedExprWithReferenceMapAndConstantFolding) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); builder.flat_expr_builder().AddAstTransform( @@ -1213,7 +1214,7 @@ TEST(FlatExprBuilderTest, ComprehensionWorksForError) { Expr expr; SourceInfo source_info; // {}[0].all(x, x) should evaluate OK but return an error value - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( id: 4 comprehension_expr { iter_var: "x" @@ -1278,7 +1279,7 @@ TEST(FlatExprBuilderTest, ComprehensionWorksForError) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -1295,7 +1296,7 @@ TEST(FlatExprBuilderTest, ComprehensionWorksForNonContainer) { Expr expr; SourceInfo source_info; // 0.all(x, x) should evaluate OK but return an error value. - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( id: 4 comprehension_expr { iter_var: "x" @@ -1349,7 +1350,7 @@ TEST(FlatExprBuilderTest, ComprehensionWorksForNonContainer) { } } })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); @@ -1721,7 +1722,7 @@ TEST(FlatExprBuilderTest, NameCollisionWithComprehensionVarLeadingDot) { TEST(FlatExprBuilderTest, MapFieldPresence) { Expr expr; SourceInfo source_info; - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( id: 1, select_expr{ operand { @@ -1731,7 +1732,7 @@ TEST(FlatExprBuilderTest, MapFieldPresence) { field: "string_int32_map" test_only: true })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_OK_AND_ASSIGN(auto cel_expr, @@ -1765,7 +1766,7 @@ TEST(FlatExprBuilderTest, MapFieldPresence) { TEST(FlatExprBuilderTest, RepeatedFieldPresence) { Expr expr; SourceInfo source_info; - google::protobuf::TextFormat::ParseFromString(R"( + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(R"( id: 1, select_expr{ operand { @@ -1775,7 +1776,7 @@ TEST(FlatExprBuilderTest, RepeatedFieldPresence) { field: "int32_list" test_only: true })", - &expr); + &expr)); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); ASSERT_OK_AND_ASSIGN(auto cel_expr, @@ -2900,6 +2901,248 @@ TEST(FlatExprBuilderTest, BlockNested) { HasSubstr("multiple cel.@block are not allowed"))); } +struct VariadicLogicalEvalTestCase { + std::string label; + std::string expr; + std::string a_val; + std::string b_val; + std::string c_val; + std::string expected_type; // "bool", "error", "unknown" + bool expected_bool = false; +}; + +class FlatExprBuilderVariadicLogicalTest + : public testing::TestWithParam {}; + +TEST_P(FlatExprBuilderVariadicLogicalTest, Evaluate) { + const auto& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(test_case.expr)); + + cel::RuntimeOptions options; + options.unknown_processing = + cel::UnknownProcessingOptions::kAttributeAndFunction; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_OK_AND_ASSIGN(auto cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + Activation activation; + google::protobuf::Arena arena; + std::vector unknown_patterns; + + // Set up variables: + auto insert_value = [&](absl::string_view name, const std::string& val) { + if (val == "true") { + activation.InsertValue(name, CelValue::CreateBool(true)); + } else if (val == "false") { + activation.InsertValue(name, CelValue::CreateBool(false)); + } else if (val == "error") { + activation.InsertValue(name, CreateErrorValue(&arena, "test error")); + } else if (val == "unknown1" || val == "unknown2") { + activation.InsertValue(name, CelValue::CreateBool(true)); + unknown_patterns.push_back(CreateCelAttributePattern(name, {})); + } + }; + + insert_value("a", test_case.a_val); + insert_value("b", test_case.b_val); + insert_value("c", test_case.c_val); + + if (!unknown_patterns.empty()) { + activation.set_unknown_attribute_patterns(std::move(unknown_patterns)); + } + + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr->Evaluate(activation, &arena)); + + if (test_case.expected_type == "bool") { + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_EQ(result.BoolOrDie(), test_case.expected_bool); + } else if (test_case.expected_type == "error") { + EXPECT_TRUE(result.IsError()) << result.DebugString(); + } else if (test_case.expected_type == "unknown") { + EXPECT_TRUE(result.IsUnknownSet()) << result.DebugString(); + } +} + +INSTANTIATE_TEST_SUITE_P( + FlatExprBuilderVariadicLogicalTest, FlatExprBuilderVariadicLogicalTest, + testing::Values( + VariadicLogicalEvalTestCase{"AND_AllTrue", "a && b && c", "true", + "true", "true", "bool", true}, + VariadicLogicalEvalTestCase{"AND_ShortCircuitFalse", "a && b && c", + "true", "false", "unset", "bool", false}, + VariadicLogicalEvalTestCase{"AND_ShortCircuitFirstFalse", "a && b && c", + "false", "unset", "unset", "bool", false}, + VariadicLogicalEvalTestCase{"OR_AllFalse", "a || b || c", "false", + "false", "false", "bool", false}, + VariadicLogicalEvalTestCase{"OR_ShortCircuitTrue", "a || b || c", + "false", "true", "unset", "bool", true}, + VariadicLogicalEvalTestCase{"OR_ShortCircuitFirstTrue", "a || b || c", + "true", "unset", "unset", "bool", true}, + VariadicLogicalEvalTestCase{"AND_Error", "a && b && c", "true", "error", + "true", "error"}, + VariadicLogicalEvalTestCase{"AND_ShortCircuitBeforeError", + "a && b && c", "false", "error", "unset", + "bool", false}, + VariadicLogicalEvalTestCase{"OR_Error", "a || b || c", "false", "error", + "false", "error"}, + VariadicLogicalEvalTestCase{"OR_ShortCircuitBeforeError", "a || b || c", + "true", "error", "unset", "bool", true}, + VariadicLogicalEvalTestCase{"AND_Unknown", "a && b && c", "true", + "unknown1", "true", "unknown"}, + VariadicLogicalEvalTestCase{"AND_ShortCircuitBeforeUnknown", + "a && b && c", "false", "unknown1", "unset", + "bool", false}, + VariadicLogicalEvalTestCase{"OR_Unknown", "a || b || c", "false", + "unknown1", "false", "unknown"}, + VariadicLogicalEvalTestCase{"OR_ShortCircuitBeforeUnknown", + "a || b || c", "true", "unknown1", "unset", + "bool", true}, + VariadicLogicalEvalTestCase{"AND_UnknownAggregation", "a && b && c", + "unknown1", "unknown2", "true", "unknown"}, + VariadicLogicalEvalTestCase{"OR_UnknownAggregation", "a || b || c", + "unknown1", "unknown2", "false", "unknown"}, + VariadicLogicalEvalTestCase{"Exists_True", "[a, b, c].exists(x, x)", + "false", "false", "true", "bool", true}, + VariadicLogicalEvalTestCase{"Exists_Unknown", "[a, b, c].exists(x, x)", + "false", "unknown1", "false", "unknown"}, + VariadicLogicalEvalTestCase{"All_False", "[a, b, c].all(x, x)", "true", + "true", "false", "bool", false}, + VariadicLogicalEvalTestCase{"All_Unknown", "[a, b, c].all(x, x)", + "true", "unknown1", "true", "unknown"})); + +struct RecursionDepthTestCase { + std::string label; + std::string expr; + int max_recursion_depth; + absl::StatusCode expected_status_code; + std::string expected_error_msg; +}; + +class FlatExprBuilderRecursionDepthTest + : public testing::TestWithParam {}; + +TEST_P(FlatExprBuilderRecursionDepthTest, CheckRecursionLimit) { + const auto& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(test_case.expr)); + + cel::RuntimeOptions options; + options.max_recursion_depth = test_case.max_recursion_depth; + options.fail_on_warnings = false; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + + auto result = + builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()); + if (test_case.expected_status_code == absl::StatusCode::kOk) { + EXPECT_THAT(result, IsOk()); + } else { + EXPECT_THAT(result, StatusIs(test_case.expected_status_code, + HasSubstr(test_case.expected_error_msg))); + } +} + +INSTANTIATE_TEST_SUITE_P( + FlatExprBuilderRecursionDepthTest, FlatExprBuilderRecursionDepthTest, + testing::Values( + RecursionDepthTestCase{"AndChildLimitExceeded", "(1 + 1) && true", 1, + absl::StatusCode::kInvalidArgument, + "Maximum recursion depth of 1 exceeded"}, + RecursionDepthTestCase{"AndParentLimitExceeded", "(1 + 1) && true", 2, + absl::StatusCode::kInvalidArgument, + "Maximum recursion depth of 2 exceeded"}, + RecursionDepthTestCase{"AndLimitSuccess", "(1 + 1) && true", 3, + absl::StatusCode::kOk, ""}, + RecursionDepthTestCase{"AndLimitSuccessGenerous", "(1 + 1) && true", 10, + absl::StatusCode::kOk, ""}, + RecursionDepthTestCase{"AndLimitSuccessUnlimited", "(1 + 1) && true", + -1, absl::StatusCode::kOk, ""}, + RecursionDepthTestCase{"OrChildLimitExceeded", "(1 + 1) || true", 1, + absl::StatusCode::kInvalidArgument, + "Maximum recursion depth of 1 exceeded"}, + RecursionDepthTestCase{"OrParentLimitExceeded", "(1 + 1) || true", 2, + absl::StatusCode::kInvalidArgument, + "Maximum recursion depth of 2 exceeded"}, + RecursionDepthTestCase{"OrLimitSuccess", "(1 + 1) || true", 3, + absl::StatusCode::kOk, ""}, + RecursionDepthTestCase{"OrLimitSuccessGenerous", + "(1 + 1) || false || false || false || false || " + "(true && true && true && true && false)", + 10, absl::StatusCode::kOk, ""}, + RecursionDepthTestCase{"OrLimitSuccessUnlimited", "(1 + 1) || true", -1, + absl::StatusCode::kOk, ""}, + RecursionDepthTestCase{"AndDepthUpdateFromSubsequentArg", + "true && (1 + 1 + 1 + 1)", 4, + absl::StatusCode::kInvalidArgument, + "Maximum recursion depth of 4 exceeded"}, + RecursionDepthTestCase{"OrDepthUpdateFromSubsequentArg", + "true || (1 + 1 + 1 + 1)", 4, + absl::StatusCode::kInvalidArgument, + "Maximum recursion depth of 4 exceeded"})); + +TEST(FlatExprBuilderTest, NonRecursiveChildBlockAndError) { + ParsedExpr parsed_expr; + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr: { + call_expr: { + function: "_&&_" + args { const_expr: { bool_value: true } } + args { + call_expr: { + function: "cel.@block" + args { + list_expr { elements { const_expr: { int64_value: 1 } } } + } + args { ident_expr: { name: "@index0" } } + } + } + } + } + )pb", + &parsed_expr)); + + cel::RuntimeOptions options; + options.max_recursion_depth = 2; + options.fail_on_warnings = false; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + EXPECT_THAT( + builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()), + StatusIs(absl::StatusCode::kInternal, + HasSubstr("failed to build recursive program"))); +} + +TEST(FlatExprBuilderTest, NonRecursiveChildBlockOrError) { + ParsedExpr parsed_expr; + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr: { + call_expr: { + function: "_||_" + args { const_expr: { bool_value: true } } + args { + call_expr: { + function: "cel.@block" + args { + list_expr { elements { const_expr: { int64_value: 1 } } } + } + args { ident_expr: { name: "@index0" } } + } + } + } + } + )pb", + &parsed_expr)); + + cel::RuntimeOptions options; + options.max_recursion_depth = 2; + options.fail_on_warnings = false; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + EXPECT_THAT( + builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()), + StatusIs(absl::StatusCode::kInternal, + HasSubstr("failed to build recursive program"))); +} + } // namespace } // namespace google::api::expr::runtime diff --git a/parser/options.h b/parser/options.h index 916a941f0..719bed454 100644 --- a/parser/options.h +++ b/parser/options.h @@ -62,6 +62,10 @@ struct ParserOptions final { // Limited to field specifiers in select and message creation, // enabled by default bool enable_quoted_identifiers = true; + + // Enables parsing logical AND & OR operators as a single flat variadic call + // instead of a balanced/nested binary AST structure. + bool enable_variadic_logical_operators = false; }; } // namespace cel diff --git a/parser/parser.cc b/parser/parser.cc index 709e2fd41..6c6434319 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -552,7 +552,7 @@ class ExpressionBalancer final { // balance creates a balanced tree from the sub-terms and returns the final // Expr value. - Expr Balance(); + Expr Balance(bool enable_variadic = false); private: // balancedTree recursively balances the terms provided to a commutative @@ -577,10 +577,13 @@ void ExpressionBalancer::AddTerm(int64_t op, Expr term) { ops_.push_back(op); } -Expr ExpressionBalancer::Balance() { +Expr ExpressionBalancer::Balance(bool enable_variadic) { if (terms_.size() == 1) { return std::move(terms_[0]); } + if (enable_variadic) { + return factory_.NewCall(ops_[0], function_, std::move(terms_)); + } return BalancedTree(0, ops_.size() - 1); } @@ -620,7 +623,8 @@ class ParserVisitor final : public CelBaseVisitor, const cel::MacroRegistry& macro_registry, bool add_macro_calls = false, bool enable_optional_syntax = false, - bool enable_quoted_identifiers = false) + bool enable_quoted_identifiers = false, + bool enable_variadic_logical_operators = false) : source_(source), factory_(source_), macro_registry_(macro_registry), @@ -628,7 +632,8 @@ class ParserVisitor final : public CelBaseVisitor, max_recursion_depth_(max_recursion_depth), add_macro_calls_(add_macro_calls), enable_optional_syntax_(enable_optional_syntax), - enable_quoted_identifiers_(enable_quoted_identifiers) {} + enable_quoted_identifiers_(enable_quoted_identifiers), + enable_variadic_logical_operators_(enable_variadic_logical_operators) {} ~ParserVisitor() override = default; @@ -719,6 +724,7 @@ class ParserVisitor final : public CelBaseVisitor, const bool add_macro_calls_; const bool enable_optional_syntax_; const bool enable_quoted_identifiers_; + const bool enable_variadic_logical_operators_; }; template ParseImpl( ExprRecursionListener listener(options.max_recursion_depth); ParserVisitor visitor( source, options.max_recursion_depth, registry, options.add_macro_calls, - options.enable_optional_syntax, options.enable_quoted_identifiers); + options.enable_optional_syntax, options.enable_quoted_identifiers, + options.enable_variadic_logical_operators); lexer.removeErrorListeners(); parser.removeErrorListeners(); diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 587b63a30..33c52b1d2 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1782,6 +1782,59 @@ TEST(NewParserBuilderTest, ToBuilderPreservesStdlibAndOptionalFromOptions) { EXPECT_FALSE(ast->IsChecked()); } +struct VariadicLogicalOperatorsTestCase { + std::string input; + std::string expected_adorned_string; +}; + +class VariadicLogicalOperatorsTest + : public testing::TestWithParam {}; + +TEST_P(VariadicLogicalOperatorsTest, Parse) { + const auto& test_case = GetParam(); + auto builder = cel::NewParserBuilder(); + builder->GetOptions().enable_variadic_logical_operators = true; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.input)); + ASSERT_OK_AND_ASSIGN(auto ast, parser->Parse(*source)); + + KindAndIdAdorner kind_and_id_adorner; + ExprPrinter w(kind_and_id_adorner); + std::string adorned_string = w.Print(ast->root_expr()); + EXPECT_EQ(adorned_string, test_case.expected_adorned_string); +} + +INSTANTIATE_TEST_SUITE_P( + VariadicLogicalOperators, VariadicLogicalOperatorsTest, + testing::Values( + VariadicLogicalOperatorsTestCase{ + .input = "a && b && c && d", + .expected_adorned_string = "_&&_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " c^#4:Expr.Ident#,\n" + " d^#6:Expr.Ident#\n" + ")^#3:Expr.Call#"}, + VariadicLogicalOperatorsTestCase{ + .input = "a || b || c || d", + .expected_adorned_string = "_||_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " c^#4:Expr.Ident#,\n" + " d^#6:Expr.Ident#\n" + ")^#3:Expr.Call#"}, + VariadicLogicalOperatorsTestCase{ + .input = "a && b && (c || d || e)", + .expected_adorned_string = "_&&_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " _||_(\n" + " c^#4:Expr.Ident#,\n" + " d^#5:Expr.Ident#,\n" + " e^#7:Expr.Ident#\n" + " )^#6:Expr.Call#\n" + ")^#3:Expr.Call#"})); + TEST(ParserTest, ParseFailurePopulatesIssues) { auto builder = cel::NewParserBuilder(); ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); diff --git a/tools/cel_unparser.cc b/tools/cel_unparser.cc index 28a1187bb..741d91208 100644 --- a/tools/cel_unparser.cc +++ b/tools/cel_unparser.cc @@ -150,6 +150,8 @@ class Unparser { // - a ternary conditional operator bool IsBinaryOrTernaryOperator(const Expr& expr); + bool IsLogicalOperator(absl::string_view op); + template void Print(Ts&&... args) { absl::StrAppend(&output_, std::forward(args)...); @@ -436,6 +438,24 @@ absl::Status Unparser::VisitUnary(const Expr::Call& expr, absl::Status Unparser::VisitBinary(const Expr::Call& expr, const std::string& op) { + if (expr.args_size() < 2) { + return absl::InvalidArgumentError( + absl::StrCat("Unexpected binary: ", expr.ShortDebugString())); + } + + const auto& fun = expr.function(); + if (IsLogicalOperator(fun)) { + for (int i = 0; i < expr.args_size(); ++i) { + if (i > 0) { + Print(kSpace, op, kSpace); + } + const auto& arg = expr.args(i); + bool arg_paren = IsComplexOperatorWithRespectTo(arg, fun); + CEL_RETURN_IF_ERROR(VisitMaybeNested(arg, arg_paren)); + } + return absl::OkStatus(); + } + if (expr.args_size() != 2) { return absl::InvalidArgumentError( absl::StrCat("Unexpected binary: ", expr.ShortDebugString())); @@ -443,7 +463,6 @@ absl::Status Unparser::VisitBinary(const Expr::Call& expr, const auto& lhs = expr.args(0); const auto& rhs = expr.args(1); - const auto& fun = expr.function(); // add parens if the current operator is lower precedence than the lhs expr // operator. @@ -549,6 +568,10 @@ bool Unparser::IsBinaryOrTernaryOperator(const Expr& expr) { IsOperatorSamePrecedence(CelOperator::CONDITIONAL, expr); } +bool Unparser::IsLogicalOperator(absl::string_view op) { + return op == CelOperator::LOGICAL_AND || op == CelOperator::LOGICAL_OR; +} + } // namespace absl::StatusOr Unparse(const Expr& expr, diff --git a/tools/cel_unparser_test.cc b/tools/cel_unparser_test.cc index 4cba4ce4d..aca6e91fd 100644 --- a/tools/cel_unparser_test.cc +++ b/tools/cel_unparser_test.cc @@ -67,6 +67,22 @@ INSTANTIATE_TEST_SUITE_P( {// Empty Expr error {"", absl::InvalidArgumentError("Unsupported Expr")}, + // Logical operators with too few arguments (single argument) + { + R"pb( + call_expr { + function: "_&&_" + args { const_expr { bool_value: true } } + })pb", + absl::InvalidArgumentError("Unexpected binary")}, + { + R"pb( + call_expr { + function: "_||_" + args { const_expr { bool_value: true } } + })pb", + absl::InvalidArgumentError("Unexpected binary")}, + // Constants {"const_expr{}", absl::InvalidArgumentError("Unsupported Constant")}, {"const_expr{bool_value: true}", "true"}, @@ -619,6 +635,7 @@ TEST_P(UnparserTestTextExpr, Test) { options.add_macro_calls = true; options.enable_optional_syntax = true; options.enable_quoted_identifiers = true; + options.enable_variadic_logical_operators = true; ASSERT_OK_AND_ASSIGN(ParsedExpr result, Parse(GetParam().expr, "unparser", options)); @@ -779,6 +796,8 @@ INSTANTIATE_TEST_SUITE_P( {"has(a.`b.c`)", ""}, {"a.`b/c`", ""}, {"a.?`b/c`", ""}, + {"a && b && c && d", ""}, + {"a || b || c || d", ""}, })); } // namespace From 09dba1a9187b58e9bd7a8c74f8b6e006d16cc99f Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 8 Jun 2026 14:25:30 -0700 Subject: [PATCH 054/350] Refactor type signature generation to use TypeSpec. Refactors the internal signature generation logic to operate on `cel::TypeSpec` instead of `cel::Type`. Adds a utility to convert `cel::Type` to `cel::TypeSpec` PiperOrigin-RevId: 928770349 --- common/BUILD | 1 + common/ast/metadata.h | 4 + common/internal/BUILD | 3 +- common/internal/signature.cc | 275 ++++++++++++---------- common/internal/signature.h | 23 +- common/internal/signature_test.cc | 367 ++++++++++++++++++------------ common/type_spec_resolver.cc | 143 +++++++++++- common/type_spec_resolver.h | 3 + common/type_spec_resolver_test.cc | 27 +++ 9 files changed, 571 insertions(+), 275 deletions(-) diff --git a/common/BUILD b/common/BUILD index a016d2cb5..01710329b 100644 --- a/common/BUILD +++ b/common/BUILD @@ -53,6 +53,7 @@ cc_library( deps = [ ":ast", ":type", + ":type_kind", "//internal:status_macros", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/common/ast/metadata.h b/common/ast/metadata.h index 197790ff3..1a69b5b50 100644 --- a/common/ast/metadata.h +++ b/common/ast/metadata.h @@ -573,6 +573,10 @@ class TypeSpec { TypeSpecKind& mutable_type_kind() { return type_kind_; } + bool is_specified() const { + return !absl::holds_alternative(type_kind_); + } + bool has_dyn() const { return absl::holds_alternative(type_kind_); } diff --git a/common/internal/BUILD b/common/internal/BUILD index 48a8dfe8b..73cbf37e9 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -145,13 +145,11 @@ cc_library( deps = [ "//common:ast", "//common:type", - "//common:type_kind", "//common:type_spec_resolver", "//internal:status_macros", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", ], @@ -165,6 +163,7 @@ cc_test( "//common:ast", "//common:type", "//common:type_kind", + "//common:type_spec_resolver", "//internal:testing", "//internal:testing_descriptor_pool", "@com_google_absl//absl/base:no_destructor", diff --git a/common/internal/signature.cc b/common/internal/signature.cc index 5c75225f9..fe315bb04 100644 --- a/common/internal/signature.cc +++ b/common/internal/signature.cc @@ -26,11 +26,9 @@ #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" #include "absl/types/optional.h" #include "common/ast.h" #include "common/type.h" -#include "common/type_kind.h" #include "common/type_spec_resolver.h" #include "internal/status_macros.h" #include "google/protobuf/arena.h" @@ -64,125 +62,145 @@ void AppendEscaped(std::string* result, std::string_view str, bool escape_dot) { } } -absl::Status AppendTypeParameters(std::string* result, const Type& type); +absl::Status AppendTypeDesc(std::string* result, const TypeSpec& type_spec); -// Recursively appends a string representation of the given `type` to `result`. -// Type parameters are enclosed in angle brackets and separated by commas. -// -// Grammar: -// TypeDesc = NamespaceIdentifier [ "<" TypeList ">" ] ; -// NamespaceIdentifier = [ "." ] Identifier { "." Identifier } ; -// TypeList = TypeElem { "," TypeElem } ; -// TypeElem = TypeDesc | TypeParam -// TypeParam = "~" Alpha ; -// Identifier = ( Alpha | "_" ) { AlphaNumeric | "_" } ; -// (* Terminals *) -// Alpha = "a"..."z" | "A"..."Z" ; -// Digit = "0"..."9" ; -// AlphaNumeric = Alpha | Digit ; -// -// For compatibility, the implementation allows unexpected characters in -// type names and parameters and escapes them with a backslash. -absl::Status AppendTypeDesc(std::string* result, const Type& type) { - switch (type.kind()) { - case TypeKind::kNull: - absl::StrAppend(result, "null"); - break; - case TypeKind::kBool: - absl::StrAppend(result, "bool"); - break; - case TypeKind::kInt: - absl::StrAppend(result, "int"); - break; - case TypeKind::kUint: - absl::StrAppend(result, "uint"); - break; - case TypeKind::kDouble: - absl::StrAppend(result, "double"); - break; - case TypeKind::kString: - absl::StrAppend(result, "string"); - break; - case TypeKind::kBytes: - absl::StrAppend(result, "bytes"); - break; - case TypeKind::kDuration: - absl::StrAppend(result, "duration"); - break; - case TypeKind::kTimestamp: - absl::StrAppend(result, "timestamp"); - break; - case TypeKind::kAny: - absl::StrAppend(result, "any"); - break; - case TypeKind::kDyn: - absl::StrAppend(result, "dyn"); - break; - case TypeKind::kBoolWrapper: - absl::StrAppend(result, "bool_wrapper"); - break; - case TypeKind::kIntWrapper: - absl::StrAppend(result, "int_wrapper"); - break; - case TypeKind::kUintWrapper: - absl::StrAppend(result, "uint_wrapper"); - break; - case TypeKind::kDoubleWrapper: - absl::StrAppend(result, "double_wrapper"); - break; - case TypeKind::kStringWrapper: - absl::StrAppend(result, "string_wrapper"); - break; - case TypeKind::kBytesWrapper: - absl::StrAppend(result, "bytes_wrapper"); - break; - case TypeKind::kList: - absl::StrAppend(result, "list"); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kMap: - absl::StrAppend(result, "map"); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kFunction: - absl::StrAppend(result, "function"); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kType: - absl::StrAppend(result, "type"); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kTypeParam: - absl::StrAppend(result, "~"); - AppendEscaped(result, type.GetTypeParam().name(), /*escape_dot=*/true); - break; - case TypeKind::kOpaque: - AppendEscaped(result, type.name(), /*escape_dot=*/false); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - case TypeKind::kStruct: - AppendEscaped(result, type.name(), /*escape_dot=*/false); - CEL_RETURN_IF_ERROR(AppendTypeParameters(result, type)); - break; - default: - return absl::InvalidArgumentError( - absl::StrFormat("Type kind: %s is not supported in CEL declarations", - type.DebugString())); +absl::Status AppendTypeSpecList(std::string* result, + const std::vector& params) { + if (!params.empty()) { + result->push_back('<'); + for (size_t i = 0; i < params.size(); ++i) { + CEL_RETURN_IF_ERROR(AppendTypeDesc(result, params[i])); + if (i < params.size() - 1) { + result->push_back(','); + } + } + result->push_back('>'); } return absl::OkStatus(); } -absl::Status AppendTypeParameters(std::string* result, const Type& type) { - const auto& parameters = type.GetParameters(); - if (!parameters.empty()) { - result->push_back('<'); - for (size_t i = 0; i < parameters.size(); ++i) { - CEL_RETURN_IF_ERROR(AppendTypeDesc(result, parameters[i])); - if (i < parameters.size() - 1) { - result->push_back(','); - } +absl::Status AppendTypeDesc(std::string* result, const TypeSpec& type_spec) { + if (type_spec.has_null()) { + absl::StrAppend(result, "null"); + } else if (type_spec.has_dyn()) { + absl::StrAppend(result, "dyn"); + } else if (type_spec.has_primitive()) { + switch (type_spec.primitive()) { + case PrimitiveType::kBool: + absl::StrAppend(result, "bool"); + break; + case PrimitiveType::kInt64: + absl::StrAppend(result, "int"); + break; + case PrimitiveType::kUint64: + absl::StrAppend(result, "uint"); + break; + case PrimitiveType::kDouble: + absl::StrAppend(result, "double"); + break; + case PrimitiveType::kString: + absl::StrAppend(result, "string"); + break; + case PrimitiveType::kBytes: + absl::StrAppend(result, "bytes"); + break; + default: + return absl::InvalidArgumentError("Unsupported primitive type"); + } + } else if (type_spec.has_well_known()) { + switch (type_spec.well_known()) { + case WellKnownTypeSpec::kAny: + absl::StrAppend(result, "any"); + break; + case WellKnownTypeSpec::kTimestamp: + absl::StrAppend(result, "timestamp"); + break; + case WellKnownTypeSpec::kDuration: + absl::StrAppend(result, "duration"); + break; + default: + return absl::InvalidArgumentError("Unsupported well-known type"); } + } else if (type_spec.has_wrapper()) { + switch (type_spec.wrapper()) { + case PrimitiveType::kBool: + absl::StrAppend(result, "bool_wrapper"); + break; + case PrimitiveType::kInt64: + absl::StrAppend(result, "int_wrapper"); + break; + case PrimitiveType::kUint64: + absl::StrAppend(result, "uint_wrapper"); + break; + case PrimitiveType::kDouble: + absl::StrAppend(result, "double_wrapper"); + break; + case PrimitiveType::kString: + absl::StrAppend(result, "string_wrapper"); + break; + case PrimitiveType::kBytes: + absl::StrAppend(result, "bytes_wrapper"); + break; + default: + return absl::InvalidArgumentError("Unsupported wrapper type"); + } + } else if (type_spec.has_list_type()) { + absl::StrAppend(result, "list<"); + if (type_spec.list_type().elem_type().is_specified()) { + CEL_RETURN_IF_ERROR( + AppendTypeDesc(result, type_spec.list_type().elem_type())); + } else { + absl::StrAppend(result, "dyn"); + } + result->push_back('>'); + } else if (type_spec.has_map_type()) { + absl::StrAppend(result, "map<"); + if (type_spec.map_type().key_type().is_specified()) { + CEL_RETURN_IF_ERROR( + AppendTypeDesc(result, type_spec.map_type().key_type())); + } else { + absl::StrAppend(result, "dyn"); + } + result->push_back(','); + if (type_spec.map_type().value_type().is_specified()) { + CEL_RETURN_IF_ERROR( + AppendTypeDesc(result, type_spec.map_type().value_type())); + } else { + absl::StrAppend(result, "dyn"); + } + result->push_back('>'); + } else if (type_spec.has_function()) { + absl::StrAppend(result, "function<"); + if (type_spec.function().result_type().is_specified()) { + CEL_RETURN_IF_ERROR( + AppendTypeDesc(result, type_spec.function().result_type())); + } else { + absl::StrAppend(result, "dyn"); + } + for (const auto& arg : type_spec.function().arg_types()) { + result->push_back(','); + CEL_RETURN_IF_ERROR(AppendTypeDesc(result, arg)); + } + result->push_back('>'); + } else if (type_spec.has_type()) { + absl::StrAppend(result, "type"); + result->push_back('<'); + CEL_RETURN_IF_ERROR(AppendTypeDesc(result, type_spec.type())); result->push_back('>'); + } else if (type_spec.has_type_param()) { + absl::StrAppend(result, "~"); + AppendEscaped(result, type_spec.type_param().type(), /*escape_dot=*/true); + } else if (type_spec.has_abstract_type()) { + AppendEscaped(result, type_spec.abstract_type().name(), + /*escape_dot=*/false); + CEL_RETURN_IF_ERROR(AppendTypeSpecList( + result, type_spec.abstract_type().parameter_types())); + } else if (type_spec.has_message_type()) { + AppendEscaped(result, type_spec.message_type().type(), + /*escape_dot=*/false); + } else { + return absl::InvalidArgumentError(absl::StrCat( + "Unsupported type in signature: ", FormatTypeSpec(type_spec))); } return absl::OkStatus(); } @@ -190,13 +208,32 @@ absl::Status AppendTypeParameters(std::string* result, const Type& type) { absl::StatusOr MakeTypeSignature(const Type& type) { std::string result; - CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, type)); + CEL_ASSIGN_OR_RETURN(TypeSpec type_spec, ConvertTypeToTypeSpec(type)); + CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, type_spec)); + return result; +} + +absl::StatusOr MakeTypeSpecSignature(const TypeSpec& type_spec) { + std::string result; + CEL_RETURN_IF_ERROR(AppendTypeDesc(&result, type_spec)); return result; } absl::StatusOr MakeOverloadSignature( std::string_view function_name, const std::vector& args, bool is_member) { + std::vector arg_type_specs; + arg_type_specs.reserve(args.size()); + for (const auto& arg : args) { + CEL_ASSIGN_OR_RETURN(TypeSpec type_spec, ConvertTypeToTypeSpec(arg)); + arg_type_specs.push_back(type_spec); + } + return MakeOverloadSignature(function_name, arg_type_specs, is_member); +} + +absl::StatusOr MakeOverloadSignature( + std::string_view function_name, const std::vector& args, + bool is_member) { std::string result; if (is_member) { if (!args.empty()) { @@ -589,10 +626,14 @@ absl::StatusOr ParseFunctionSignature( return out; } +absl::StatusOr ParseTypeSpec(std::string_view signature) { + std::string stripped_sig = StripUnescapedWhitespace(signature); + return ParseTypeSignature(stripped_sig); +} + absl::StatusOr ParseType(std::string_view signature, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& pool) { - std::string stripped_sig = StripUnescapedWhitespace(signature); - CEL_ASSIGN_OR_RETURN(auto type_spec, ParseTypeSignature(stripped_sig)); + CEL_ASSIGN_OR_RETURN(auto type_spec, ParseTypeSpec(signature)); return cel::ConvertTypeSpecToType(type_spec, arena, pool); } diff --git a/common/internal/signature.h b/common/internal/signature.h index 3fdba4b2e..8a44fbd5c 100644 --- a/common/internal/signature.h +++ b/common/internal/signature.h @@ -27,7 +27,7 @@ namespace cel::common_internal { -// Generates an signature for a `cel::Type`, which is a string representation of +// Generates a signature for a `cel::Type`, which is a string representation of // the type. // // Examples: @@ -37,7 +37,17 @@ namespace cel::common_internal { // - `list>` absl::StatusOr MakeTypeSignature(const Type& type); -// Generates an identifier for a function overload based on the function name +// Generates a signature for a `cel::TypeSpec`, which is a string +// representation of the type. +// +// Examples: +// +// - `int` +// - `list` +// - `list>` +absl::StatusOr MakeTypeSpecSignature(const TypeSpec& type_spec); + +// Generates a signature for a function overload based on the function name // and the types of the arguments. If `is_member` is true, the first argument // type is used as the receiver and is prepended to the function name, followed // by a dollar sign. @@ -59,6 +69,15 @@ absl::StatusOr MakeOverloadSignature( std::string_view function_name, const std::vector& args, bool is_member); +// Generates a signature for a function overload based on the function name +// and the type specs of the arguments. See above for more details. +absl::StatusOr MakeOverloadSignature( + std::string_view function_name, const std::vector& args, + bool is_member); + +// Parses a string type signature directly into a `cel::TypeSpec`. +absl::StatusOr ParseTypeSpec(std::string_view signature); + // Parses a string type signature directly into a `cel::Type`. absl::StatusOr ParseType(std::string_view signature, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& pool); diff --git a/common/internal/signature_test.cc b/common/internal/signature_test.cc index 765055f75..17b628d88 100644 --- a/common/internal/signature_test.cc +++ b/common/internal/signature_test.cc @@ -14,6 +14,7 @@ // limitations under the License. #include +#include #include #include @@ -24,6 +25,7 @@ #include "common/ast.h" #include "common/type.h" #include "common/type_kind.h" +#include "common/type_spec_resolver.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "google/protobuf/arena.h" @@ -42,82 +44,9 @@ google::protobuf::Arena* GetTestArena() { return &*arena; } -void VerifyParsedMatchesType(const TypeSpec& parsed, const Type& original) { - switch (original.kind()) { - case TypeKind::kDyn: - EXPECT_TRUE(parsed.has_dyn()); - break; - case TypeKind::kNull: - EXPECT_TRUE(parsed.has_null()); - break; - case TypeKind::kBool: - EXPECT_EQ(parsed.primitive(), PrimitiveType::kBool); - break; - case TypeKind::kInt: - EXPECT_EQ(parsed.primitive(), PrimitiveType::kInt64); - break; - case TypeKind::kUint: - EXPECT_EQ(parsed.primitive(), PrimitiveType::kUint64); - break; - case TypeKind::kDouble: - EXPECT_EQ(parsed.primitive(), PrimitiveType::kDouble); - break; - case TypeKind::kString: - EXPECT_EQ(parsed.primitive(), PrimitiveType::kString); - break; - case TypeKind::kBytes: - EXPECT_EQ(parsed.primitive(), PrimitiveType::kBytes); - break; - case TypeKind::kAny: - EXPECT_EQ(parsed.well_known(), WellKnownTypeSpec::kAny); - break; - case TypeKind::kTimestamp: - EXPECT_EQ(parsed.well_known(), WellKnownTypeSpec::kTimestamp); - break; - case TypeKind::kDuration: - EXPECT_EQ(parsed.well_known(), WellKnownTypeSpec::kDuration); - break; - case TypeKind::kList: - EXPECT_TRUE(parsed.has_list_type()); - if (!original.GetParameters().empty()) { - VerifyParsedMatchesType(parsed.list_type().elem_type(), - original.GetParameters()[0]); - } - break; - case TypeKind::kMap: - EXPECT_TRUE(parsed.has_map_type()); - if (!original.GetParameters().empty()) { - VerifyParsedMatchesType(parsed.map_type().key_type(), - original.GetParameters()[0]); - } - if (original.GetParameters().size() > 1) { - VerifyParsedMatchesType(parsed.map_type().value_type(), - original.GetParameters()[1]); - } - break; - case TypeKind::kBoolWrapper: - case TypeKind::kIntWrapper: - case TypeKind::kUintWrapper: - case TypeKind::kDoubleWrapper: - case TypeKind::kStringWrapper: - case TypeKind::kBytesWrapper: - EXPECT_TRUE(parsed.has_wrapper()); - break; - case TypeKind::kType: - EXPECT_TRUE(parsed.has_type()); - if (!original.GetParameters().empty()) { - VerifyParsedMatchesType(parsed.type(), original.GetParameters()[0]); - } - break; - case TypeKind::kTypeParam: - EXPECT_TRUE(parsed.has_type_param()); - break; - default: - EXPECT_TRUE(parsed.has_abstract_type()); - break; - } +void VerifyParsedMatchesType(const TypeSpec& parsed, const TypeSpec& expected) { + EXPECT_EQ(parsed, expected); } - void VerifyTypesEqual(const Type& lhs, const Type& rhs) { EXPECT_EQ(lhs.kind(), rhs.kind()); if (lhs.kind() != rhs.kind()) return; @@ -138,7 +67,7 @@ void VerifyTypesEqual(const Type& lhs, const Type& rhs) { } struct TypeSignatureTestCase { - Type type; + TypeSpec type; std::string expected_signature; std::string expected_error; }; @@ -149,104 +78,208 @@ TEST_P(TypeSignatureTest, TypeSignature) { const auto& param = GetParam(); absl::StatusOr signature = - common_internal::MakeTypeSignature(param.type); + common_internal::MakeTypeSpecSignature(param.type); if (!param.expected_error.empty()) { EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr(param.expected_error))); } else { EXPECT_THAT(signature, IsOkAndHolds(param.expected_signature)); + + absl::StatusOr type = ConvertTypeSpecToType( + param.type, GetTestArena(), *GetTestingDescriptorPool()); + ASSERT_THAT(type, ::absl_testing::IsOk()); + EXPECT_THAT(MakeTypeSignature(*type), + IsOkAndHolds(param.expected_signature)); } } std::vector GetTypeSignatureTestCases() { return { { - .type = StringType{}, - .expected_signature = "string", + .type = TypeSpec(NullTypeSpec{}), + .expected_signature = "null", + }, + { + .type = TypeSpec(PrimitiveType::kBool), + .expected_signature = "bool", }, { - .type = IntType{}, + .type = TypeSpec(PrimitiveType::kInt64), .expected_signature = "int", }, { - .type = ListType(GetTestArena(), StringType{}), - .expected_signature = "list", + .type = TypeSpec(PrimitiveType::kUint64), + .expected_signature = "uint", }, { - .type = TypeType(GetTestArena(), IntType{}), - .expected_signature = "type", + .type = TypeSpec(PrimitiveType::kDouble), + .expected_signature = "double", + }, + { + .type = TypeSpec(PrimitiveType::kString), + .expected_signature = "string", }, { - .type = ListType(GetTestArena(), TypeParamType("A")), + .type = TypeSpec(PrimitiveType::kBytes), + .expected_signature = "bytes", + }, + { + .type = TypeSpec( + MessageTypeSpec("cel.expr.conformance.proto3.TestAllTypes")), + .expected_signature = "cel.expr.conformance.proto3.TestAllTypes", + }, + { + .type = TypeSpec( + AbstractType("cel.expr.conformance.proto3.TestAllTypes", {})), + .expected_signature = "cel.expr.conformance.proto3.TestAllTypes", + }, + { + .type = TypeSpec(WellKnownTypeSpec::kDuration), + .expected_signature = "duration", + }, + { + .type = TypeSpec(WellKnownTypeSpec::kTimestamp), + .expected_signature = "timestamp", + }, + { + .type = TypeSpec( + ListTypeSpec(std::make_unique(PrimitiveType::kString))), + .expected_signature = "list", + }, + { + .type = TypeSpec( + ListTypeSpec(std::make_unique(ParamTypeSpec("A")))), .expected_signature = "list<~A>", }, { - .type = ListType(GetTestArena(), TypeParamType("A(ParamTypeSpec("A(ParamTypeSpec(R"(a,b..(d)\e)")))), + .expected_signature = R"(list<~a\,b\.\\.\(d\)\\e>)", + }, + { + .type = TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kInt64), + std::make_unique(DynTypeSpec()))), .expected_signature = "map", }, { - .type = - MapType(GetTestArena(), TypeParamType("B"), TypeParamType("C")), + .type = TypeSpec( + MapTypeSpec(std::make_unique(ParamTypeSpec("B")), + std::make_unique(ParamTypeSpec("C")))), .expected_signature = "map<~B,~C>", }, { - .type = OpaqueType(GetTestArena(), "bar", - {FunctionType(GetTestArena(), TypeParamType("D"), - {StringType{}, BoolType{}})}), - .expected_signature = "bar>", + .type = TypeSpec(MapTypeSpec( + std::make_unique(PrimitiveType::kInt64), nullptr)), + .expected_signature = "map", + }, + { + .type = TypeSpec(MapTypeSpec(nullptr, nullptr)), + .expected_signature = "map", + }, + { + .type = TypeSpec(std::make_unique(PrimitiveType::kInt64)), + .expected_signature = "type", }, { - .type = AnyType{}, + .type = TypeSpec(WellKnownTypeSpec::kAny), .expected_signature = "any", }, { - .type = DurationType{}, - .expected_signature = "duration", + .type = TypeSpec(DynTypeSpec{}), + .expected_signature = "dyn", }, { - .type = TimestampType{}, - .expected_signature = "timestamp", + .type = TypeSpec(AbstractType( + "bar", {TypeSpec(FunctionTypeSpec( + std::make_unique(ParamTypeSpec("D")), + {TypeSpec(PrimitiveType::kString), + TypeSpec(PrimitiveType::kBool)}))})), + .expected_signature = "bar>", + }, + { + .type = + TypeSpec(AbstractType("bar", {TypeSpec(PrimitiveType::kInt64), + TypeSpec(PrimitiveType::kString)})), + .expected_signature = "bar", }, { - .type = BoolWrapperType{}, + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)), .expected_signature = "bool_wrapper", }, { - .type = IntWrapperType{}, + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)), .expected_signature = "int_wrapper", }, { - .type = UintWrapperType{}, + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)), .expected_signature = "uint_wrapper", }, { - .type = MessageType(GetTestingDescriptorPool()->FindMessageTypeByName( - "cel.expr.conformance.proto3.TestAllTypes")), - .expected_signature = "cel.expr.conformance.proto3.TestAllTypes", + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)), + .expected_signature = "double_wrapper", }, { - .type = ListType(GetTestArena(), TypeParamType(R"(a,b..(d)\e)")), - .expected_signature = R"(list<~a\,b\.\\.\(d\)\\e>)", + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)), + .expected_signature = "string_wrapper", + }, + { + .type = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)), + .expected_signature = "bytes_wrapper", + }, + { + .type = TypeSpec( + FunctionTypeSpec(nullptr, {TypeSpec(PrimitiveType::kInt64)})), + .expected_signature = "function", + }, + { + .type = TypeSpec(FunctionTypeSpec( + std::make_unique(PrimitiveType::kInt64), {})), + .expected_signature = "function", + }, + { + .type = TypeSpec(FunctionTypeSpec(nullptr, {})), + .expected_signature = "function", }, }; } +INSTANTIATE_TEST_SUITE_P(TypeSignatureTest, TypeSignatureTest, + ValuesIn(GetTypeSignatureTestCases())); + TEST(TypeSignatureTest, UnsupportedTypes) { EXPECT_THAT(common_internal::MakeTypeSignature(UnknownType{}), StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Type kind: *unknown* is not supported"))); + HasSubstr("Unsupported Type kind: *unknown*"))); EXPECT_THAT(common_internal::MakeTypeSignature(ErrorType{}), StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Type kind: *error* is not supported"))); -} + HasSubstr("Unsupported type in signature: *error*"))); -INSTANTIATE_TEST_SUITE_P(TypeIdTest, TypeSignatureTest, - ValuesIn(GetTypeSignatureTestCases())); + EXPECT_THAT(common_internal::MakeTypeSpecSignature( + TypeSpec(static_cast(999))), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported primitive type"))); + + EXPECT_THAT(common_internal::MakeTypeSpecSignature( + TypeSpec(static_cast(999))), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported well-known type"))); + + EXPECT_THAT(common_internal::MakeTypeSpecSignature(TypeSpec( + PrimitiveTypeWrapper(static_cast(999)))), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported wrapper type"))); +} TEST_P(TypeSignatureTest, ParseTypeCheck) { const auto& param = GetParam(); @@ -254,13 +287,16 @@ TEST_P(TypeSignatureTest, ParseTypeCheck) { auto parsed = ParseType(param.expected_signature, GetTestArena(), *GetTestingDescriptorPool()); ASSERT_THAT(parsed, ::absl_testing::IsOk()); - VerifyTypesEqual(*parsed, param.type); + ASSERT_OK_AND_ASSIGN(auto expected_type, + ConvertTypeSpecToType(param.type, GetTestArena(), + *GetTestingDescriptorPool())); + VerifyTypesEqual(*parsed, expected_type); } } struct OverloadSignatureTestCase { std::string function_name = "hello"; - std::vector args; + std::vector args; bool is_member = false; std::string expected_signature; std::string expected_error; @@ -285,98 +321,110 @@ TEST_P(OverloadSignatureTest, OverloadSignature) { std::vector GetOverloadSignatureTestCases() { return { { - .args = {StringType{}}, + .args = {TypeSpec(PrimitiveType::kString)}, .expected_signature = "hello(string)", }, { - .args = {IntType{}, UintType{}}, + .args = {TypeSpec(PrimitiveType::kInt64), + TypeSpec(PrimitiveType::kUint64)}, .expected_signature = "hello(int,uint)", }, { - .args = {ListType(GetTestArena(), StringType{})}, + .args = {TypeSpec(ListTypeSpec( + std::make_unique(PrimitiveType::kString)))}, .expected_signature = "hello(list)", }, { - .args = {ListType(GetTestArena(), TypeParamType("A"))}, + .args = {TypeSpec( + ListTypeSpec(std::make_unique(ParamTypeSpec("A"))))}, .expected_signature = "hello(list<~A>)", }, { - .args = {MapType(GetTestArena(), IntType{}, DynType{})}, + .args = {TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kInt64), + std::make_unique(DynTypeSpec())))}, .expected_signature = "hello(map)", }, { - .args = {MapType(GetTestArena(), TypeParamType("B"), - TypeParamType("C"))}, + .args = {TypeSpec( + MapTypeSpec(std::make_unique(ParamTypeSpec("B")), + std::make_unique(ParamTypeSpec("C"))))}, .expected_signature = "hello(map<~B,~C>)", }, + { - .args = {OpaqueType( - GetTestArena(), "bar", - {FunctionType(GetTestArena(), TypeParamType("D"), {})})}, + .args = {TypeSpec(AbstractType( + "bar", + {TypeSpec(FunctionTypeSpec( + std::make_unique(ParamTypeSpec("D")), {}))}))}, .expected_signature = "hello(bar>)", }, { - .args = {AnyType{}}, + .args = {TypeSpec(WellKnownTypeSpec::kAny)}, .expected_signature = "hello(any)", }, { - .args = {DurationType{}}, + .args = {TypeSpec(WellKnownTypeSpec::kDuration)}, .expected_signature = "hello(duration)", }, { - .args = {TimestampType{}}, + .args = {TypeSpec(WellKnownTypeSpec::kTimestamp)}, .expected_signature = "hello(timestamp)", }, { - .args = {BoolWrapperType{}}, + .args = {TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool))}, .expected_signature = "hello(bool_wrapper)", }, { - .args = {IntWrapperType{}}, + .args = {TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64))}, .expected_signature = "hello(int_wrapper)", }, { - .args = {UintWrapperType{}}, + .args = {TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64))}, .expected_signature = "hello(uint_wrapper)", }, { - .args = {MessageType( - GetTestingDescriptorPool()->FindMessageTypeByName( - "cel.expr.conformance.proto3.TestAllTypes"))}, + .args = {TypeSpec( + AbstractType("cel.expr.conformance.proto3.TestAllTypes", {}))}, .expected_signature = "hello(cel.expr.conformance.proto3.TestAllTypes)", }, { - .args = {StringType{}}, + .args = {TypeSpec(PrimitiveType::kString)}, .is_member = true, .expected_signature = "string.hello()", }, { - .args = {StringType{}, ListType(GetTestArena(), BoolType{})}, + .args = {TypeSpec(PrimitiveType::kString), + TypeSpec(ListTypeSpec( + std::make_unique(PrimitiveType::kBool)))}, .is_member = true, .expected_signature = "string.hello(list)", }, { - .args = {StringType{}, BoolType{}, DynType{}}, + .args = {TypeSpec(PrimitiveType::kString), + TypeSpec(PrimitiveType::kBool), TypeSpec(DynTypeSpec())}, .is_member = true, .expected_signature = "string.hello(bool,dyn)", }, { .function_name = "hello", - .args = {OpaqueType(GetTestArena(), "bar", - {TypeParamType("dummy.type")})}, + .args = {TypeSpec( + AbstractType("bar", {TypeSpec(ParamTypeSpec("dummy.type"))}))}, .is_member = true, .expected_signature = R"(bar<~dummy\.type>.hello())", }, { .function_name = "inspect", - .args = {Type(TypeType(GetTestArena(), StringType{}))}, + .args = {TypeSpec( + std::make_unique(PrimitiveType::kString))}, .expected_signature = "inspect(type)", }, { .function_name = R"(h.(e),l\o)", - .args = {StringType{}, - ListType(GetTestArena(), TypeParamType(R"(a,b..(d)\e)"))}, + .args = {TypeSpec(PrimitiveType::kString), + TypeSpec(ListTypeSpec(std::make_unique( + ParamTypeSpec(R"(a,b..(d)\e)"))))}, .is_member = true, .expected_signature = R"(string.h\.\(e\)\,l\\\o(list<~a\,b\.\\.\(d\)\\e>))", @@ -385,7 +433,8 @@ std::vector GetOverloadSignatureTestCases() { } TEST(OverloadSignatureTest, MemberFunctionNoReceiverError) { - auto signature = common_internal::MakeOverloadSignature("hello", {}, true); + auto signature = common_internal::MakeOverloadSignature( + "hello", std::vector{}, true); EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Member function with no receiver"))); @@ -564,8 +613,15 @@ TEST(ParseSignatureTest, ParsingErrors) { EXPECT_THAT(ParseFunctionSignature("foo"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Invalid function signature"))); + EXPECT_THAT( + ParseType("list b < c>", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("mismatched brackets"))); - // Parameter count validations for list and map types. + // Parameter count validations for list, map and type types. EXPECT_THAT(ParseType("list", GetTestArena(), *GetTestingDescriptorPool()), StatusIs(absl::StatusCode::kInvalidArgument, @@ -578,6 +634,18 @@ TEST(ParseSignatureTest, ParsingErrors) { *GetTestingDescriptorPool()), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("map expects 0 or 2 parameters"))); + EXPECT_THAT(ParseType("type", GetTestArena(), + *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("type expects at most 1 parameter"))); + + // Invalid parameter name validations. + EXPECT_THAT(ParseType("~", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid type parameter name"))); + EXPECT_THAT(ParseType("~A", GetTestArena(), *GetTestingDescriptorPool()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid type parameter name"))); // Enforcing valid function and identifier names. EXPECT_THAT(ParseFunctionSignature("()"), @@ -700,5 +768,20 @@ TEST(ParseSignatureTest, EmptyOrWhitespaceErrors) { HasSubstr("Empty type signature"))); } +TEST(OverloadSignatureTest, ArgumentTypeVector) { + std::vector args; + args.push_back(Type(IntType())); + args.push_back(Type(StringType())); + args.push_back(Type(ListType(GetTestArena(), IntType()))); + args.push_back( + Type(MessageType(GetTestingDescriptorPool()->FindMessageTypeByName( + "cel.expr.conformance.proto3.TestAllTypes")))); + args.push_back(Type(OpaqueType(GetTestArena(), "Foo", {TypeParamType("T")}))); + ASSERT_OK_AND_ASSIGN(auto sig, MakeOverloadSignature("foo", args, false)); + EXPECT_EQ(sig, + "foo(int,string,list,cel.expr.conformance.proto3.TestAllTypes," + "Foo<~T>)"); +} + } // namespace } // namespace cel::common_internal diff --git a/common/type_spec_resolver.cc b/common/type_spec_resolver.cc index 97451f390..90c9930a8 100644 --- a/common/type_spec_resolver.cc +++ b/common/type_spec_resolver.cc @@ -14,6 +14,7 @@ #include "common/type_spec_resolver.h" +#include #include #include #include @@ -22,8 +23,12 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "common/ast.h" #include "common/type.h" +#include "common/type_kind.h" #include "internal/status_macros.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" namespace cel { @@ -85,28 +90,42 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, } if (type_spec.has_list_type()) { - CEL_ASSIGN_OR_RETURN( - auto elem_type, - ConvertTypeSpecToType(type_spec.list_type().elem_type(), arena, pool)); + Type elem_type; + if (type_spec.list_type().elem_type().is_specified()) { + CEL_ASSIGN_OR_RETURN( + elem_type, ConvertTypeSpecToType(type_spec.list_type().elem_type(), + arena, pool)); + } return Type(ListType(arena, elem_type)); } if (type_spec.has_map_type()) { - CEL_ASSIGN_OR_RETURN( - auto key_type, - ConvertTypeSpecToType(type_spec.map_type().key_type(), arena, pool)); - CEL_ASSIGN_OR_RETURN( - auto value_type, - ConvertTypeSpecToType(type_spec.map_type().value_type(), arena, pool)); + Type key_type; + if (type_spec.map_type().key_type().is_specified()) { + CEL_ASSIGN_OR_RETURN( + key_type, + ConvertTypeSpecToType(type_spec.map_type().key_type(), arena, pool)); + } + + Type value_type; + if (type_spec.map_type().value_type().is_specified()) { + CEL_ASSIGN_OR_RETURN( + value_type, ConvertTypeSpecToType(type_spec.map_type().value_type(), + arena, pool)); + } return Type(MapType(arena, key_type, value_type)); } if (type_spec.has_function()) { const auto& func_spec = type_spec.function(); - CEL_ASSIGN_OR_RETURN( - auto result_type, - ConvertTypeSpecToType(func_spec.result_type(), arena, pool)); + Type result_type; + if (func_spec.result_type().is_specified()) { + CEL_ASSIGN_OR_RETURN( + result_type, + ConvertTypeSpecToType(func_spec.result_type(), arena, pool)); + } std::vector arg_types; + arg_types.reserve(func_spec.arg_types().size()); for (const auto& arg_spec : func_spec.arg_types()) { CEL_ASSIGN_OR_RETURN(auto arg_type, ConvertTypeSpecToType(arg_spec, arena, pool)); @@ -179,4 +198,104 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, return absl::InvalidArgumentError("Unknown TypeSpec kind"); } +absl::StatusOr ConvertTypeToTypeSpec(const Type& type) { + switch (type.kind()) { + case TypeKind::kNull: + return TypeSpec(NullTypeSpec{}); + case TypeKind::kDyn: + return TypeSpec(DynTypeSpec{}); + case TypeKind::kBool: + return TypeSpec(PrimitiveType::kBool); + case TypeKind::kInt: + return TypeSpec(PrimitiveType::kInt64); + case TypeKind::kUint: + return TypeSpec(PrimitiveType::kUint64); + case TypeKind::kDouble: + return TypeSpec(PrimitiveType::kDouble); + case TypeKind::kString: + return TypeSpec(PrimitiveType::kString); + case TypeKind::kBytes: + return TypeSpec(PrimitiveType::kBytes); + case TypeKind::kAny: + return TypeSpec(WellKnownTypeSpec::kAny); + case TypeKind::kTimestamp: + return TypeSpec(WellKnownTypeSpec::kTimestamp); + case TypeKind::kDuration: + return TypeSpec(WellKnownTypeSpec::kDuration); + case TypeKind::kBoolWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)); + case TypeKind::kIntWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)); + case TypeKind::kUintWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)); + case TypeKind::kDoubleWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)); + case TypeKind::kStringWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)); + case TypeKind::kBytesWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)); + case TypeKind::kList: { + CEL_ASSIGN_OR_RETURN(auto elem_type, + ConvertTypeToTypeSpec(type.GetList().element())); + return TypeSpec( + ListTypeSpec(std::make_unique(std::move(elem_type)))); + } + case TypeKind::kMap: { + CEL_ASSIGN_OR_RETURN(auto key_type, + ConvertTypeToTypeSpec(type.GetMap().key())); + CEL_ASSIGN_OR_RETURN(auto value_type, + ConvertTypeToTypeSpec(type.GetMap().value())); + return TypeSpec( + MapTypeSpec(std::make_unique(std::move(key_type)), + std::make_unique(std::move(value_type)))); + } + case TypeKind::kFunction: { + auto func_type = type.GetFunction(); + CEL_ASSIGN_OR_RETURN(auto result_type, + ConvertTypeToTypeSpec(func_type.result())); + std::vector arg_types; + arg_types.reserve(func_type.args().size()); + for (const auto& arg : func_type.args()) { + CEL_ASSIGN_OR_RETURN(auto arg_type, ConvertTypeToTypeSpec(arg)); + arg_types.push_back(std::move(arg_type)); + } + return TypeSpec( + FunctionTypeSpec(std::make_unique(std::move(result_type)), + std::move(arg_types))); + } + case TypeKind::kTypeParam: + return TypeSpec(ParamTypeSpec(std::string(type.GetTypeParam().name()))); + case TypeKind::kStruct: { + if (type.IsMessage()) { + return TypeSpec(MessageTypeSpec(std::string(type.GetMessage().name()))); + } + return absl::InvalidArgumentError("Unsupported struct type"); + } + case TypeKind::kOpaque: { + auto opaque_type = type.GetOpaque(); + std::vector params; + params.reserve(opaque_type.GetParameters().size()); + for (const auto& param : opaque_type.GetParameters()) { + CEL_ASSIGN_OR_RETURN(auto param_type, ConvertTypeToTypeSpec(param)); + params.push_back(std::move(param_type)); + } + return TypeSpec( + AbstractType(std::string(opaque_type.name()), std::move(params))); + } + case TypeKind::kType: { + CEL_ASSIGN_OR_RETURN(auto nested_type, + ConvertTypeToTypeSpec(type.GetType().GetType())); + return TypeSpec(std::make_unique(std::move(nested_type))); + } + case TypeKind::kError: + return TypeSpec(ErrorTypeSpec::kValue); + case TypeKind::kEnum: + return TypeSpec( + AbstractType(std::string(type.GetEnum().name()), /*params=*/{})); + default: + return absl::InvalidArgumentError(absl::StrCat( + "Unsupported Type kind: ", TypeKindToString(type.kind()))); + } +} + } // namespace cel diff --git a/common/type_spec_resolver.h b/common/type_spec_resolver.h index 44e1e088f..edbfa3bde 100644 --- a/common/type_spec_resolver.h +++ b/common/type_spec_resolver.h @@ -32,6 +32,9 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& pool); +// Resolves a `cel::Type` to a `cel::TypeSpec`. +absl::StatusOr ConvertTypeToTypeSpec(const Type& type); + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_TYPE_SPEC_RESOLVER_H_ diff --git a/common/type_spec_resolver_test.cc b/common/type_spec_resolver_test.cc index c7fbb2cf8..1cda7280f 100644 --- a/common/type_spec_resolver_test.cc +++ b/common/type_spec_resolver_test.cc @@ -23,6 +23,7 @@ #include "absl/base/no_destructor.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" +#include "common/ast.h" #include "common/type.h" #include "common/type_kind.h" #include "internal/testing.h" @@ -33,6 +34,7 @@ namespace cel { namespace { using ::absl_testing::IsOk; +using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::cel::internal::GetTestingDescriptorPool; using ::testing::HasSubstr; @@ -67,6 +69,7 @@ TEST_P(ConversionTest, TestTypeSpecConversion) { auto t, ConvertTypeSpecToType(std::get<0>(GetParam()), GetTestArena(), *GetTestingDescriptorPool())); EXPECT_EQ(t.kind(), std::get<1>(GetParam())); + EXPECT_THAT(ConvertTypeToTypeSpec(t), IsOkAndHolds(std::get<0>(GetParam()))); } INSTANTIATE_TEST_SUITE_P( @@ -104,6 +107,8 @@ TEST(TypeSpecResolverTest, ListTypeConversion) { ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsList()); EXPECT_TRUE(t->GetList().element().IsInt()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, MapTypeConversion) { @@ -116,6 +121,8 @@ TEST(TypeSpecResolverTest, MapTypeConversion) { EXPECT_TRUE(t->IsMap()); EXPECT_TRUE(t->GetMap().key().IsString()); EXPECT_TRUE(t->GetMap().value().IsBytes()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, FunctionTypeConversion) { @@ -129,6 +136,8 @@ TEST(TypeSpecResolverTest, FunctionTypeConversion) { EXPECT_TRUE(t->IsFunction()); EXPECT_EQ(t->GetFunction().args().size(), 1); EXPECT_TRUE(t->GetFunction().result().IsBool()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, TypeParamConversion) { @@ -138,6 +147,8 @@ TEST(TypeSpecResolverTest, TypeParamConversion) { ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsTypeParam()); EXPECT_EQ(t->GetTypeParam().name(), "T"); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, MessageTypeConversion) { @@ -148,6 +159,10 @@ TEST(TypeSpecResolverTest, MessageTypeConversion) { ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsMessage()); EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes"); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ( + spec2, + TypeSpec(MessageTypeSpec("cel.expr.conformance.proto3.TestAllTypes"))); } TEST(TypeSpecResolverTest, MessageTypeWithParamsError) { @@ -172,6 +187,8 @@ TEST(TypeSpecResolverTest, UnresolvedAbstractTypeFallbackToOpaque) { EXPECT_EQ(t->name(), "my.custom.OpaqueType"); EXPECT_EQ(t->GetParameters().size(), 1); EXPECT_TRUE(t->GetParameters()[0].IsInt()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, OptionalType) { @@ -186,6 +203,8 @@ TEST(TypeSpecResolverTest, OptionalType) { EXPECT_EQ(t->GetParameters().size(), 1); EXPECT_TRUE(t->GetParameters()[0].IsInt()); EXPECT_TRUE(t->IsOptional()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, TypeTypeConversion) { @@ -196,6 +215,8 @@ TEST(TypeSpecResolverTest, TypeTypeConversion) { ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsType()); EXPECT_TRUE(t->GetType().GetType().IsInt()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, ErrorTypeConversion) { @@ -204,6 +225,8 @@ TEST(TypeSpecResolverTest, ErrorTypeConversion) { ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsError()); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, MessageTypeSpecConversion) { @@ -213,6 +236,8 @@ TEST(TypeSpecResolverTest, MessageTypeSpecConversion) { ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsMessage()); EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes"); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, MessageTypeSpecNotFoundError) { @@ -231,6 +256,8 @@ TEST(TypeSpecResolverTest, EnumTypeConversion) { ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsEnum()); EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"); + ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); + EXPECT_EQ(spec2, spec); } TEST(TypeSpecResolverTest, EnumTypeWithParamsError) { From f5d0d5ff13e770bd35be79e5ce7e81fe35f63e91 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 8 Jun 2026 15:38:29 -0700 Subject: [PATCH 055/350] Clean up unused dependencies in cel/cpp/common BUILD files. Remove unused dependencies from cel/cpp/common/BUILD and cel/cpp/common/internal/BUILD. PiperOrigin-RevId: 928808099 --- common/BUILD | 9 --------- common/internal/BUILD | 2 -- 2 files changed, 11 deletions(-) diff --git a/common/BUILD b/common/BUILD index 01710329b..f7c897e57 100644 --- a/common/BUILD +++ b/common/BUILD @@ -403,7 +403,6 @@ cc_library( ":allocator", ":arena", ":data", - ":native_type", ":reference_count", "//common/internal:metadata", "//common/internal:reference_count", @@ -425,13 +424,9 @@ cc_test( ":allocator", ":data", ":memory", - ":native_type", "//common/internal:reference_count", "//internal:testing", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/debugging:leak_check", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", ], @@ -1024,9 +1019,6 @@ cc_library( deps = [ ":decl", ":decl_proto", - ":type", - ":type_proto", - "//internal:status_macros", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -1194,6 +1186,5 @@ cc_test( ":container", "//internal:testing", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:status_matchers", ], ) diff --git a/common/internal/BUILD b/common/internal/BUILD index 73cbf37e9..b07faf229 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -21,10 +21,8 @@ cc_library( name = "casting", hdrs = ["casting.h"], deps = [ - "//common:native_type", "//internal:casts", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/types:optional", ], From 87fea87272429b9c85655caeda7ab5e4804bf5f4 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 8 Jun 2026 22:27:41 -0700 Subject: [PATCH 056/350] Add support for type signatures in CEL environment YAML configuration. The `env_yaml` parser now accepts type signatures for variable types and function overload signatures. The `type` field can be used instead of `type_name` for variables, allowing a more compact representation of types, including type parameters and parameterized types. The `signature` field can be used for function overloads, providing a single string to define the overload's target, arguments, and member status. The `return` type in function overloads can now also be specified as a type signature string. PiperOrigin-RevId: 928959415 --- env/BUILD | 7 +- env/env_yaml.cc | 311 +++++++++++++++++++++++++--------- env/env_yaml.h | 37 +++- env/env_yaml_test.cc | 381 +++++++++++++++++++++++++++++++++--------- env/type_info.cc | 226 +++++++++++++++++++++++++ env/type_info.h | 7 + env/type_info_test.cc | 169 +++++++++++++++++++ 7 files changed, 978 insertions(+), 160 deletions(-) diff --git a/env/BUILD b/env/BUILD index 41ffc1723..3035e11ac 100644 --- a/env/BUILD +++ b/env/BUILD @@ -28,6 +28,7 @@ cc_library( "type_info.h", ], deps = [ + "//common:ast", "//common:constant", "//common:type", "//common:type_kind", @@ -120,7 +121,9 @@ cc_library( features = ["-use_header_modules"], deps = [ ":config", + "//common:ast", "//common:constant", + "//common/internal:signature", "//internal:status_macros", "//internal:strings", "@com_google_absl//absl/algorithm:container", @@ -178,9 +181,11 @@ cc_test( ":config", "//common:type", "//common:type_proto", + "//common/ast:metadata", "//internal:proto_matchers", "//internal:testing", "//internal:testing_descriptor_pool", + "@com_google_absl//absl/status", "@com_google_protobuf//:protobuf", ], ) @@ -201,7 +206,6 @@ cc_test( "//common:type", "//common:value", "//compiler", - "//internal:proto_matchers", "//internal:status_macros", "//internal:testing", "//internal:testing_descriptor_pool", @@ -241,7 +245,6 @@ cc_test( "//common:value", "//compiler", "//extensions:math_ext", - "//internal:status_macros", "//internal:testing", "//internal:testing_descriptor_pool", "//runtime", diff --git a/env/env_yaml.cc b/env/env_yaml.cc index 159786598..8c635e65f 100644 --- a/env/env_yaml.cc +++ b/env/env_yaml.cc @@ -35,8 +35,11 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" +#include "common/ast.h" #include "common/constant.h" +#include "common/internal/signature.h" #include "env/config.h" +#include "env/type_info.h" #include "internal/status_macros.h" #include "internal/strings.h" #include "yaml-cpp/emitter.h" @@ -117,8 +120,8 @@ absl::StatusOr GetBinary(absl::string_view yaml, return binary; } else { return YamlError(yaml, node, - "Node '" + GetString(yaml, node) + - "' is not a valid Base64 encoded binary"); + absl::StrCat("Node '", GetString(yaml, node), + "' is not a valid Base64 encoded binary")); } } @@ -131,10 +134,22 @@ absl::StatusOr GetBool(absl::string_view yaml, absl::string_view key, return node.as(); } catch (YAML::Exception& e) { return YamlError(yaml, node, - "Node '" + std::string(key) + "' is not a boolean"); + absl::StrCat("Node '", key, "' is not a boolean")); } } +// Returns the key in the map `node` that has the given `value_node` as its +// value. If no such key exists, returns `value_node` itself. +YAML::Node GetContextNodeForKeyValue(const YAML::Node& node, + const YAML::Node& value_node) { + for (const auto& kv : node) { + if (kv.second.IsDefined() && kv.second.is(value_node)) { + return kv.first; + } + } + return value_node; +} + absl::Status ParseName(Config& config, absl::string_view yaml, const YAML::Node& root) { const YAML::Node name = root["name"]; @@ -407,7 +422,23 @@ absl::Status ParseStandardLibraryConfig(Config& config, absl::string_view yaml, absl::StatusOr ParseTypeInfo(const YAML::Node& node, absl::string_view yaml) { Config::TypeInfo type_config; + const YAML::Node type = node["type"]; const YAML::Node type_name = node["type_name"]; + if (type.IsDefined() && type_name.IsDefined()) { + return YamlError(yaml, GetContextNodeForKeyValue(node, type_name), + "Node 'type' and 'type_name' are mutually exclusive"); + } + + if (type.IsDefined()) { + if (!type.IsScalar()) { + return YamlError(yaml, type, "Node 'type' is not a string"); + } + CEL_ASSIGN_OR_RETURN(auto type_spec, + common_internal::ParseTypeSpec(GetString(yaml, type))); + CEL_ASSIGN_OR_RETURN(auto type_config, TypeSpecToTypeInfo(type_spec)); + return type_config; + } + if (!type_name.IsDefined()) { return type_config; } @@ -627,7 +658,8 @@ absl::Status ParseVariableConfigs(Config& config, absl::string_view yaml, } absl::StatusOr ParseFunctionOverloadConfig( - absl::string_view yaml, const YAML::Node& overload) { + absl::string_view yaml, const YAML::Node& overload, + absl::string_view function_name) { Config::FunctionOverloadConfig overload_config; if (!overload || !overload.IsMap()) { return YamlError(yaml, overload, "Function overload is not a map"); @@ -654,40 +686,89 @@ absl::StatusOr ParseFunctionOverloadConfig( } } + const YAML::Node signature_node = overload["signature"]; const YAML::Node target = overload["target"]; - if (target.IsDefined()) { - if (!target.IsMap()) { - return YamlError(yaml, target, "Function overload target is not a map"); + const YAML::Node args = overload["args"]; + if (signature_node.IsDefined()) { + if (!signature_node.IsScalar()) { + return YamlError(yaml, signature_node, + "Function overload signature is not a string"); } - CEL_ASSIGN_OR_RETURN(Config::TypeInfo type_info, - ParseTypeInfo(target, yaml)); - overload_config.is_member_function = true; - overload_config.parameters.push_back(type_info); - } - const YAML::Node args = overload["args"]; - if (args.IsDefined()) { - if (!args.IsSequence()) { - return YamlError(yaml, args, "Function overload args is not a sequence"); + if (target.IsDefined()) { + return YamlError(yaml, GetContextNodeForKeyValue(overload, target), + "Function overload signature and target are mutually " + "exclusive"); + } + if (args.IsDefined()) { + return YamlError(yaml, GetContextNodeForKeyValue(overload, args), + "Function overload signature and args are mutually " + "exclusive"); + } + + std::string signature = GetString(yaml, signature_node); + CEL_ASSIGN_OR_RETURN( + common_internal::ParsedFunctionOverload parsed_signature, + common_internal::ParseFunctionSignature(signature)); + if (parsed_signature.function_name != function_name) { + return YamlError(yaml, signature_node, + absl::StrCat("Function overload name \"", + parsed_signature.function_name, + "\" does not match function name \"", + function_name, "\"")); + } + overload_config.is_member_function = parsed_signature.is_member; + if (!parsed_signature.signature_type.has_function()) { + return absl::InternalError(absl::StrCat( + "Function overload signature has no function type: ", signature)); } - for (const YAML::Node& arg : args) { - if (!arg.IsMap()) { - return YamlError(yaml, arg, "Function overload arg is not a map"); + const FunctionTypeSpec& function_type_spec = + parsed_signature.signature_type.function(); + for (const auto& arg : function_type_spec.arg_types()) { + CEL_ASSIGN_OR_RETURN(auto type_info, TypeSpecToTypeInfo(arg)); + overload_config.parameters.push_back(std::move(type_info)); + } + } else { + if (target.IsDefined()) { + if (!target.IsMap()) { + return YamlError(yaml, target, "Function overload target is not a map"); } CEL_ASSIGN_OR_RETURN(Config::TypeInfo type_info, - ParseTypeInfo(arg, yaml)); + ParseTypeInfo(target, yaml)); + overload_config.is_member_function = true; overload_config.parameters.push_back(type_info); } - } + if (args.IsDefined()) { + if (!args.IsSequence()) { + return YamlError(yaml, args, + "Function overload args is not a sequence"); + } + for (const YAML::Node& arg : args) { + if (!arg.IsMap()) { + return YamlError(yaml, arg, "Function overload arg is not a map"); + } + CEL_ASSIGN_OR_RETURN(Config::TypeInfo type_info, + ParseTypeInfo(arg, yaml)); + overload_config.parameters.push_back(type_info); + } + } + } const YAML::Node return_type = overload["return"]; if (return_type.IsDefined()) { - if (!return_type.IsMap()) { - return YamlError(yaml, return_type, - "Function overload return type is not a map"); + if (return_type.IsScalar()) { + CEL_ASSIGN_OR_RETURN(auto type_spec, common_internal::ParseTypeSpec( + GetString(yaml, return_type))); + CEL_ASSIGN_OR_RETURN(overload_config.return_type, + TypeSpecToTypeInfo(type_spec)); + } else if (return_type.IsMap()) { + CEL_ASSIGN_OR_RETURN(overload_config.return_type, + ParseTypeInfo(return_type, yaml)); + } else { + return YamlError( + yaml, return_type, + "Function overload return type is neither a string nor a map"); } - CEL_ASSIGN_OR_RETURN(overload_config.return_type, - ParseTypeInfo(return_type, yaml)); } return overload_config; } @@ -728,8 +809,9 @@ absl::Status ParseFunctionConfigs(Config& config, absl::string_view yaml, } for (const YAML::Node& overload : overloads) { - CEL_ASSIGN_OR_RETURN(Config::FunctionOverloadConfig overload_config, - ParseFunctionOverloadConfig(yaml, overload)); + CEL_ASSIGN_OR_RETURN( + Config::FunctionOverloadConfig overload_config, + ParseFunctionOverloadConfig(yaml, overload, function_config.name)); function_config.overload_configs.push_back(std::move(overload_config)); } } @@ -893,26 +975,43 @@ void EmitStandardLibraryConfig(const Config& env_config, YAML::Emitter& out) { out << YAML::EndMap; } -void EmitTypeInfo(const Config::TypeInfo& type_info, YAML::Emitter& out) { +void EmitTypeInfo(const Config::TypeInfo& type_info, YAML::Emitter& out, + const EnvConfigToYamlOptions& options) { // Note: the map is already started when this is called, so we don't emit // BeginMap here or EndMap at the end. - out << YAML::Key << "type_name"; - out << YAML::Value << YAML::DoubleQuoted << type_info.name; - if (type_info.is_type_param) { - out << YAML::Key << "is_type_param" << YAML::Value << true; - } - if (!type_info.params.empty()) { - out << YAML::Key << "params" << YAML::Value << YAML::BeginSeq; - for (const Config::TypeInfo& param : type_info.params) { - out << YAML::BeginMap; - EmitTypeInfo(param, out); - out << YAML::EndMap; + bool signature_generated = false; + if (options.use_type_signatures) { + absl::StatusOr type_spec = TypeInfoToTypeSpec(type_info); + if (type_spec.ok()) { + absl::StatusOr signature = + common_internal::MakeTypeSpecSignature(*type_spec); + if (signature.ok()) { + out << YAML::Key << "type"; + out << YAML::Value << YAML::DoubleQuoted << *signature; + signature_generated = true; + } + } + } + if (!signature_generated) { + out << YAML::Key << "type_name"; + out << YAML::Value << YAML::DoubleQuoted << type_info.name; + if (type_info.is_type_param) { + out << YAML::Key << "is_type_param" << YAML::Value << true; + } + if (!type_info.params.empty()) { + out << YAML::Key << "params" << YAML::Value << YAML::BeginSeq; + for (const Config::TypeInfo& param : type_info.params) { + out << YAML::BeginMap; + EmitTypeInfo(param, out, options); + out << YAML::EndMap; + } + out << YAML::EndSeq; } - out << YAML::EndSeq; } } -void EmitVariableConfigs(const Config& env_config, YAML::Emitter& out) { +void EmitVariableConfigs(const Config& env_config, YAML::Emitter& out, + const EnvConfigToYamlOptions& options) { const auto& variable_configs = env_config.GetVariableConfigs(); if (variable_configs.empty()) { return; @@ -936,7 +1035,7 @@ void EmitVariableConfigs(const Config& env_config, YAML::Emitter& out) { out << YAML::Key << "description"; out << YAML::Value << YAML::DoubleQuoted << variable_config.description; } - EmitTypeInfo(variable_config.type_info, out); + EmitTypeInfo(variable_config.type_info, out, options); if (variable_config.value.has_value()) { const Constant& constant = variable_config.value; switch (constant.kind_case()) { @@ -991,51 +1090,97 @@ void EmitVariableConfigs(const Config& env_config, YAML::Emitter& out) { } void EmitFunctionOverloadConfig( - const Config::FunctionOverloadConfig& overload_config, YAML::Emitter& out) { + absl::string_view function_name, + const Config::FunctionOverloadConfig& overload_config, YAML::Emitter& out, + const EnvConfigToYamlOptions& options) { out << YAML::BeginMap; - out << YAML::Key << "id"; - out << YAML::Value << YAML::DoubleQuoted << overload_config.overload_id; - if (overload_config.is_member_function) { - out << YAML::Key << "target" << YAML::Value; - out << YAML::BeginMap; - if (overload_config.parameters.empty()) { - // This should never happen, but if it does, emit a dynamic type. - EmitTypeInfo({.name = "dyn"}, out); - } else { - EmitTypeInfo(overload_config.parameters[0], out); + if (!overload_config.overload_id.empty()) { + out << YAML::Key << "id"; + out << YAML::Value << YAML::DoubleQuoted << overload_config.overload_id; + } + bool signature_generated = false; + if (options.use_type_signatures) { + bool param_type_spec_generated = true; + std::vector params; + params.reserve(overload_config.parameters.size()); + for (const auto& parameter : overload_config.parameters) { + absl::StatusOr type_spec = TypeInfoToTypeSpec(parameter); + if (!type_spec.ok()) { + param_type_spec_generated = false; + break; + } + params.push_back(std::move(*type_spec)); } - out << YAML::EndMap; - if (overload_config.parameters.size() > 1) { - out << YAML::Key << "args"; - out << YAML::Value << YAML::BeginSeq; - for (size_t i = 1; i < overload_config.parameters.size(); ++i) { - out << YAML::BeginMap; - EmitTypeInfo(overload_config.parameters[i], out); - out << YAML::EndMap; + if (param_type_spec_generated) { + absl::StatusOr signature = + common_internal::MakeOverloadSignature( + function_name, params, overload_config.is_member_function); + if (signature.ok()) { + out << YAML::Key << "signature"; + out << YAML::Value << YAML::DoubleQuoted << *signature; + signature_generated = true; } - out << YAML::EndSeq; } - } else { - if (!overload_config.parameters.empty()) { - out << YAML::Key << "args"; - out << YAML::Value << YAML::BeginSeq; - for (const Config::TypeInfo& parameter : overload_config.parameters) { - out << YAML::BeginMap; - EmitTypeInfo(parameter, out); - out << YAML::EndMap; + } + if (!signature_generated) { + if (overload_config.is_member_function) { + out << YAML::Key << "target" << YAML::Value; + out << YAML::BeginMap; + if (overload_config.parameters.empty()) { + // This should never happen, but if it does, emit a dynamic type. + EmitTypeInfo({.name = "dyn"}, out, options); + } else { + EmitTypeInfo(overload_config.parameters[0], out, options); + } + out << YAML::EndMap; + if (overload_config.parameters.size() > 1) { + out << YAML::Key << "args"; + out << YAML::Value << YAML::BeginSeq; + for (size_t i = 1; i < overload_config.parameters.size(); ++i) { + out << YAML::BeginMap; + EmitTypeInfo(overload_config.parameters[i], out, options); + out << YAML::EndMap; + } + out << YAML::EndSeq; + } + } else { + if (!overload_config.parameters.empty()) { + out << YAML::Key << "args"; + out << YAML::Value << YAML::BeginSeq; + for (const Config::TypeInfo& parameter : overload_config.parameters) { + out << YAML::BeginMap; + EmitTypeInfo(parameter, out, options); + out << YAML::EndMap; + } + out << YAML::EndSeq; } - out << YAML::EndSeq; } } - out << YAML::Key << "return"; - out << YAML::Value << YAML::BeginMap; - EmitTypeInfo(overload_config.return_type, out); - out << YAML::EndMap; - + bool return_type_signature_generated = false; + if (options.use_type_signatures) { + absl::StatusOr type_spec = + TypeInfoToTypeSpec(overload_config.return_type); + if (type_spec.ok()) { + absl::StatusOr signature = + common_internal::MakeTypeSpecSignature(*type_spec); + if (signature.ok()) { + out << YAML::Key << "return"; + out << YAML::Value << YAML::DoubleQuoted << *signature; + return_type_signature_generated = true; + } + } + } + if (!return_type_signature_generated) { + out << YAML::Key << "return"; + out << YAML::Value << YAML::BeginMap; + EmitTypeInfo(overload_config.return_type, out, options); + out << YAML::EndMap; + } out << YAML::EndMap; } -void EmitFunctionConfigs(const Config& env_config, YAML::Emitter& out) { +void EmitFunctionConfigs(const Config& env_config, YAML::Emitter& out, + const EnvConfigToYamlOptions& options) { const std::vector& function_configs = env_config.GetFunctionConfigs(); if (function_configs.empty()) { @@ -1085,7 +1230,8 @@ void EmitFunctionConfigs(const Config& env_config, YAML::Emitter& out) { out << YAML::Key << "overloads" << YAML::Value << YAML::BeginSeq; for (const Config::FunctionOverloadConfig& overload_config : sorted_overloads) { - EmitFunctionOverloadConfig(overload_config, out); + EmitFunctionOverloadConfig(function_config.name, overload_config, out, + options); } out << YAML::EndSeq; } @@ -1116,7 +1262,8 @@ absl::StatusOr EnvConfigFromYaml(const std::string& yaml) { return config; } -void EnvConfigToYaml(const Config& env_config, std::ostream& os) { +void EnvConfigToYaml(const Config& env_config, std::ostream& os, + const EnvConfigToYamlOptions& options) { YAML::Emitter out(os); out.SetIndent(2); out << YAML::BeginMap; @@ -1127,8 +1274,8 @@ void EnvConfigToYaml(const Config& env_config, std::ostream& os) { EmitContainerConfig(env_config, out); EmitExtensionConfigs(env_config, out); EmitStandardLibraryConfig(env_config, out); - EmitVariableConfigs(env_config, out); - EmitFunctionConfigs(env_config, out); + EmitVariableConfigs(env_config, out, options); + EmitFunctionConfigs(env_config, out, options); out << YAML::EndMap; } diff --git a/env/env_yaml.h b/env/env_yaml.h index c96b45933..7bf7bf6b4 100644 --- a/env/env_yaml.h +++ b/env/env_yaml.h @@ -31,8 +31,43 @@ namespace cel { // expensive expressions. absl::StatusOr EnvConfigFromYaml(const std::string& yaml); +struct EnvConfigToYamlOptions { + // Whether to use type and overload signatures instead of arg/return types in + // the output YAML. + // Example of type signature: "map>" vs + // type_name: "map" + // params: + // - type_name: "int" + // - type_name: "A" + // params: + // - type_name: "B" + // is_type_param: true + // + // Example of overload signature config: + // name: "foo" + // overloads: + // - signature: "timestamp.foo(A<~B>)" + // return: "int" + // vs + // name: "foo" + // overloads: + // - id: "foo_id" + // target: + // type_name: "timestamp" + // args: + // - type_name: "A" + // params: + // - type_name: "B" + // is_type_param: true + // return: + // type_name: "int" + // TODO(uncreated-issue/91): default to true after all dependencies are updated + bool use_type_signatures = false; +}; + // EnvConfigToYaml serializes an environment configuration as a YAML string. -void EnvConfigToYaml(const Config& env_config, std::ostream& os); +void EnvConfigToYaml(const Config& env_config, std::ostream& os, + const EnvConfigToYamlOptions& options = {}); } // namespace cel diff --git a/env/env_yaml_test.cc b/env/env_yaml_test.cc index d19c0dbfb..f6bde59c9 100644 --- a/env/env_yaml_test.cc +++ b/env/env_yaml_test.cc @@ -195,6 +195,28 @@ TEST(EnvYamlTest, ParseVariableConfigs) { } TEST(EnvYamlTest, ParseVariableConfigWithTypeParams) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + variables: + - name: "dict" + type: "map" + )yaml")); + + const Config::VariableConfig& variable_config = + config.GetVariableConfigs()[0]; + EXPECT_EQ(variable_config.name, "dict"); + const auto& type_info = variable_config.type_info; + EXPECT_EQ(type_info.name, "map"); + EXPECT_FALSE(type_info.is_type_param); + EXPECT_THAT(type_info.params, SizeIs(2)); + EXPECT_EQ(type_info.params[0].name, "string"); + EXPECT_FALSE(type_info.params[0].is_type_param); + EXPECT_THAT(type_info.params[0].params, IsEmpty()); + EXPECT_EQ(type_info.params[1].name, "A"); + EXPECT_TRUE(type_info.params[1].is_type_param); + EXPECT_THAT(type_info.params[1].params, IsEmpty()); +} + +TEST(EnvYamlTest, ParseVariableConfigWithTypeParamsLegacySyntax) { ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( variables: - name: "dict" @@ -221,7 +243,7 @@ TEST(EnvYamlTest, ParseVariableConfigWithTypeParams) { } struct ParseConstantTestCase { - std::string type_name; + std::string type; std::string value; std::string expected_error; // Empty if no error. Constant expected_constant; @@ -236,10 +258,10 @@ TEST_P(EnvYamlParseConstantTest, EnvYamlParseConstant) { R"yaml( variables: - name: "const" - type_name: "%s" + type: "%s" value: %s )yaml", - param.type_name, param.value); + param.type, param.value); absl::StatusOr status_or_config = EnvConfigFromYaml(yaml); if (!param.expected_error.empty()) { EXPECT_THAT(status_or_config, StatusIs(absl::StatusCode::kInvalidArgument, @@ -251,8 +273,7 @@ TEST_P(EnvYamlParseConstantTest, EnvYamlParseConstant) { const Config::VariableConfig& variable_config = config.GetVariableConfigs()[0]; EXPECT_EQ(variable_config.name, "const"); - EXPECT_EQ(variable_config.type_info.name, param.type_name) - << " yaml: " << yaml; + EXPECT_EQ(variable_config.type_info.name, param.type) << " yaml: " << yaml; EXPECT_EQ(variable_config.value, param.expected_constant) << " yaml: " << yaml; } @@ -260,119 +281,119 @@ TEST_P(EnvYamlParseConstantTest, EnvYamlParseConstant) { std::vector GetParseConstantTestCases() { return { ParseConstantTestCase{ - .type_name = "null", + .type = "null", .value = "\"\"", .expected_constant = Constant(nullptr), }, ParseConstantTestCase{ - .type_name = "null", + .type = "null", .value = "anything", .expected_error = "Failed to parse null constant", }, ParseConstantTestCase{ - .type_name = "bool", + .type = "bool", .value = "TRUE", .expected_constant = Constant(true), }, ParseConstantTestCase{ - .type_name = "bool", + .type = "bool", .value = "false", .expected_constant = Constant(false), }, ParseConstantTestCase{ - .type_name = "bool", + .type = "bool", .value = "yes", .expected_error = "Failed to parse bool constant", }, ParseConstantTestCase{ - .type_name = "int", + .type = "int", .value = "42", .expected_constant = Constant(int64_t{42}), }, ParseConstantTestCase{ - .type_name = "int", + .type = "int", .value = "41.999", .expected_error = "Failed to parse int constant", }, ParseConstantTestCase{ - .type_name = "uint", + .type = "uint", .value = "42", .expected_constant = Constant(uint64_t{42}), }, ParseConstantTestCase{ - .type_name = "uint", + .type = "uint", .value = "42u", .expected_constant = Constant(uint64_t{42}), }, ParseConstantTestCase{ - .type_name = "uint", + .type = "uint", .value = "-1", .expected_error = "Failed to parse uint constant", }, ParseConstantTestCase{ - .type_name = "double", + .type = "double", .value = "42.42", .expected_constant = Constant(42.42), }, ParseConstantTestCase{ - .type_name = "double", + .type = "double", .value = "abc", .expected_error = "Failed to parse double constant", }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "abc", .expected_constant = Constant(BytesConstant("abc")), }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "b\"\\xFF\\x00\\x01\"", .expected_constant = Constant(BytesConstant(absl::string_view("\xff\x00\x01", 3))), }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "!!binary /wAB", .expected_constant = Constant(BytesConstant(absl::string_view("\xff\x00\x01", 3))), }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "!!binary YWJj=", .expected_error = "Node 'YWJj=' is not a valid Base64 encoded binary", }, ParseConstantTestCase{ - .type_name = "bytes", + .type = "bytes", .value = "abc", .expected_constant = Constant(BytesConstant("abc")), }, ParseConstantTestCase{ - .type_name = "string", + .type = "string", .value = "abc", .expected_constant = Constant(StringConstant("abc")), }, ParseConstantTestCase{ - .type_name = "string", + .type = "string", .value = "\"\\\"abc\\\"\"", .expected_constant = Constant(StringConstant("\"abc\"")), }, ParseConstantTestCase{ - .type_name = "duration", + .type = "duration", .value = "1s", .expected_constant = Constant(absl::Seconds(1)), }, ParseConstantTestCase{ - .type_name = "duration", + .type = "duration", .value = "abc", .expected_error = "Failed to parse duration constant", }, ParseConstantTestCase{ - .type_name = "timestamp", + .type = "timestamp", .value = "2023-01-01T00:00:00Z", .expected_constant = Constant(absl::FromUnixSeconds(1672531200)), }, ParseConstantTestCase{ - .type_name = "timestamp", + .type = "timestamp", .value = "abc", .expected_error = "Failed to parse timestamp constant", }, @@ -439,6 +460,50 @@ TEST_P(EnvYamlParseFunctionTest, EnvYamlParseFunction) { std::vector GetParseFunctionTestCases() { return { + ParseFunctionTestCase{ + .yaml = R"yaml( + functions: + - name: "isEmpty" + description: |- + determines whether a list is empty, + or a string has no characters + overloads: + - signature: "google.protobuf.StringValue.isEmpty()" + examples: + - "''.isEmpty() // true" + return: "bool" + - signature: "list<~T>.isEmpty()" + examples: + - "[].isEmpty() // true" + - "[1].isEmpty() // false" + return: "bool" + )yaml", + .expected_function_config = + { + .name = "isEmpty", + .description = "determines whether a list is empty,\nor a " + "string has no characters", + .overload_configs = + { + Config::FunctionOverloadConfig{ + .examples = {"''.isEmpty() // true"}, + .is_member_function = true, + .parameters = {{.name = "string_wrapper"}}, + .return_type = {.name = "bool"}, + }, + Config::FunctionOverloadConfig{ + .examples = {"[].isEmpty() // true", + "[1].isEmpty() // false"}, + .is_member_function = true, + .parameters = {{.name = "list", + .params = {{.name = "T", + .is_type_param = + true}}}}, + .return_type = {.name = "bool"}, + }, + }, + }, + }, ParseFunctionTestCase{ .yaml = R"yaml( functions: @@ -495,6 +560,34 @@ std::vector GetParseFunctionTestCases() { }, }, }, + ParseFunctionTestCase{ + .yaml = R"yaml( + functions: + - name: "contains" + overloads: + - signature: "contains(list<~T>, ~T)" + examples: + - "contains([1, 2, 3], 2) // true" + return: "bool" + )yaml", + .expected_function_config = + { + .name = "contains", + .overload_configs = + { + Config::FunctionOverloadConfig{ + .examples = {"contains([1, 2, 3], 2) // true"}, + .is_member_function = false, + .parameters = + {{.name = "list", + .params = {{.name = "T", + .is_type_param = true}}}, + {.name = "T", .is_type_param = true}}, + .return_type = {.name = "bool"}, + }, + }, + }, + }, ParseFunctionTestCase{ .yaml = R"yaml( functions: @@ -865,6 +958,18 @@ INSTANTIATE_TEST_SUITE_P( "| is_type_param: maybe\n" "| ^", }, + ParseTestCase{ + .yaml = R"yaml( + variables: + - name: "foo" + type_name: "opaque" + type: "opaque" + )yaml", + .expected_error = "4:19: Node 'type' and 'type_name'" + " are mutually exclusive\n" + "| type_name: \"opaque\"\n" + "| ^", + }, ParseTestCase{ .yaml = R"yaml( variables: @@ -965,12 +1070,65 @@ INSTANTIATE_TEST_SUITE_P( - name: "foo" overloads: - id: "foo_int64" - return: "to sender" + return: [1] )yaml", .expected_error = "6:31: Function overload return type" - " is not a map\n" - "| return: \"to sender\"\n" + " is neither a string nor a map\n" + "| return: [1]\n" "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + functions: + - name: "foo" + overloads: + - id: "foo_int64" + signature: "bar()" + )yaml", + .expected_error = "6:34: Function overload name \"bar\" " + "does not match function name \"foo\"\n" + "| signature: \"bar()\"\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + functions: + - name: "foo" + overloads: + - signature: [ "foo()" ] + )yaml", + .expected_error = + "5:34: Function overload signature is not a string\n" + "| - signature: [ \"foo()\" ]\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + functions: + - name: "foo" + overloads: + - signature: "foo()" + target: + type_name: "int" + )yaml", + .expected_error = "6:23: Function overload signature and target " + "are mutually exclusive\n" + "| target:\n" + "| ^", + }, + ParseTestCase{ + .yaml = R"yaml( + functions: + - name: "foo" + overloads: + - signature: "foo()" + args: + - type_name: "int" + )yaml", + .expected_error = "6:23: Function overload signature and args are " + "mutually exclusive\n" + "| args:\n" + "| ^", })); std::string Unindent(std::string_view yaml) { @@ -999,6 +1157,7 @@ std::string Unindent(std::string_view yaml) { struct ExportTestCase { absl::StatusOr config; std::string expected_yaml; + std::string expected_alt_yaml; }; class EnvYamlExportTest : public testing::TestWithParam {}; @@ -1007,10 +1166,18 @@ TEST_P(EnvYamlExportTest, EnvYamlExport) { const ExportTestCase& param = GetParam(); ASSERT_OK_AND_ASSIGN(Config config, param.config); std::stringstream ss; - EnvConfigToYaml(config, ss); + EnvConfigToYaml(config, ss, {.use_type_signatures = true}); std::string yaml_output = Unindent(ss.str()); std::string expected_yaml = Unindent(param.expected_yaml); EXPECT_EQ(yaml_output, expected_yaml); + + if (!param.expected_alt_yaml.empty()) { + std::stringstream alt_ss; + EnvConfigToYaml(config, alt_ss, {.use_type_signatures = false}); + std::string alt_yaml_output = Unindent(alt_ss.str()); + std::string expected_alt_yaml = Unindent(param.expected_alt_yaml); + EXPECT_EQ(alt_yaml_output, expected_alt_yaml); + } } std::vector GetExportTestCases() { @@ -1211,7 +1378,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "null" + type: "null" )yaml", }, ExportTestCase{ @@ -1224,6 +1391,12 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "bool" + value: true + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "bool" @@ -1240,6 +1413,12 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "int" + value: 42 + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "int" @@ -1258,7 +1437,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "uint" + type: "uint" value: 777 )yaml", }, @@ -1274,7 +1453,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "double" + type: "double" value: 0.75 )yaml", }, @@ -1291,7 +1470,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "bytes" + type: "bytes" value: b"\xff\x00\x01" )yaml", }, @@ -1309,7 +1488,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "string" + type: "string" value: "'single' \"double\"" )yaml", }, @@ -1324,6 +1503,12 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "duration" + value: 1h2m3s + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "duration" @@ -1340,6 +1525,12 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "timestamp" + value: 2026-01-02T03:04:05Z + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "timestamp" @@ -1358,7 +1549,7 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( variables: - name: "foo" - type_name: "google.expr.proto3.test.TestAllTypes" + type: "google.expr.proto3.test.TestAllTypes" )yaml", }, ExportTestCase{ @@ -1373,6 +1564,11 @@ std::vector GetExportTestCases() { return config; }(), .expected_yaml = R"yaml( + variables: + - name: "foo" + type: "A" + )yaml", + .expected_alt_yaml = R"yaml( variables: - name: "foo" type_name: "A" @@ -1402,12 +1598,22 @@ std::vector GetExportTestCases() { {.overload_id = "foo_overload_id", .is_member_function = true, .parameters = {{.name = "timestamp"}, - {.name = "A", .params = {{.name = "B"}}}}, + {.name = "A", + .params = {{.name = "B", + .is_type_param = true}}}}, .return_type = {.name = "int"}}, }})); return config; }(), .expected_yaml = R"yaml( + functions: + - name: "foo" + overloads: + - id: "foo_overload_id" + signature: "timestamp.foo(A<~B>)" + return: "int" + )yaml", + .expected_alt_yaml = R"yaml( functions: - name: "foo" overloads: @@ -1418,6 +1624,7 @@ std::vector GetExportTestCases() { - type_name: "A" params: - type_name: "B" + is_type_param: true return: type_name: "int" )yaml", @@ -1427,6 +1634,7 @@ std::vector GetExportTestCases() { Config config; CEL_RETURN_IF_ERROR(config.AddFunctionConfig( {.name = "foo", + .description = "my desc", .overload_configs = { {.overload_id = "foo_overload_a", .parameters = {{.name = "timestamp"}}, @@ -1442,6 +1650,19 @@ std::vector GetExportTestCases() { .expected_yaml = R"yaml( functions: - name: "foo" + description: "my desc" + overloads: + - id: "foo_overload_b" + signature: "foo(double,A)" + return: "string" + - id: "foo_overload_a" + signature: "foo(timestamp)" + return: "list" + )yaml", + .expected_alt_yaml = R"yaml( + functions: + - name: "foo" + description: "my desc" overloads: - id: "foo_overload_b" args: @@ -1466,9 +1687,10 @@ std::vector GetExportTestCases() { INSTANTIATE_TEST_SUITE_P(EnvYamlExportTest, EnvYamlExportTest, ::testing::ValuesIn(GetExportTestCases())); -class EnvYamlRoundTripTest : public testing::TestWithParam {}; +class EnvYamlStructuredRoundTripTest + : public testing::TestWithParam {}; -TEST_P(EnvYamlRoundTripTest, EnvYamlRoundTrip) { +TEST_P(EnvYamlStructuredRoundTripTest, EnvYamlRoundTrip) { const std::string& yaml = Unindent(GetParam()); ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(yaml)); @@ -1477,7 +1699,7 @@ TEST_P(EnvYamlRoundTripTest, EnvYamlRoundTrip) { EXPECT_EQ(ss.str(), yaml); } -std::vector GetRoundTripTestCases() { +std::vector GetStructuredRoundTripTestCases() { return { R"yaml( stdlib: @@ -1536,74 +1758,83 @@ std::vector GetRoundTripTestCases() { overloads: - id: "string_to_timestamp" )yaml", + R"yaml( + functions: + - name: "bar" + - name: "foo" + )yaml", + }; +} + +INSTANTIATE_TEST_SUITE_P( + EnvYamlStructuredRoundTripTest, EnvYamlStructuredRoundTripTest, + ::testing::ValuesIn(GetStructuredRoundTripTestCases())); + +class EnvYamlSignatureRoundTripTest + : public testing::TestWithParam {}; + +TEST_P(EnvYamlSignatureRoundTripTest, EnvYamlRoundTrip) { + const std::string& yaml = Unindent(GetParam()); + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(yaml)); + + std::stringstream ss; + EnvConfigToYaml(config, ss, {.use_type_signatures = true}); + EXPECT_EQ(ss.str(), yaml); +} + +std::vector GetSignatureRoundTripTestCases() { + return { R"yaml( variables: - name: "a" - type_name: "null" + type: "null" - name: "b" - type_name: "bool" + type: "bool" value: true - name: "c" - type_name: "int" + type: "int" value: 42 - name: "d" - type_name: "uint" + type: "uint" value: 777 - name: "e" - type_name: "double" + type: "double" value: 0.75 - name: "f" - type_name: "bytes" + type: "bytes" value: b"\xff\x00\x01" - name: "g" - type_name: "string" + type: "string" value: "plain 'single' \"double\"" - name: "h" - type_name: "duration" + type: "duration" value: 1h2m3s - name: "i" - type_name: "timestamp" + type: "timestamp" value: 2026-01-02T03:04:05Z )yaml", - R"yaml( - functions: - - name: "bar" - - name: "foo" - )yaml", R"yaml( functions: - name: "foo" overloads: - id: "foo_overload_id" - target: - type_name: "timestamp" - args: - - type_name: "A" - params: - - type_name: "B" - return: - type_name: "int" + signature: "timestamp.foo(A<~B>)" + return: "int" )yaml", R"yaml( functions: - name: "foo" overloads: - id: "foo_overload_id" - args: - - type_name: "timestamp" - - type_name: "A" - params: - - type_name: "B" - return: - type_name: "list" - params: - - type_name: "int" + signature: "foo(timestamp,A<~B>)" + return: "list" )yaml", }; } -INSTANTIATE_TEST_SUITE_P(EnvYamlRoundTripTest, EnvYamlRoundTripTest, - ::testing::ValuesIn(GetRoundTripTestCases())); +INSTANTIATE_TEST_SUITE_P(EnvYamlSignatureRoundTripTest, + EnvYamlSignatureRoundTripTest, + ::testing::ValuesIn(GetSignatureRoundTripTestCases())); } // namespace } // namespace cel diff --git a/env/type_info.cc b/env/type_info.cc index a5b47b6f1..f49fab9f4 100644 --- a/env/type_info.cc +++ b/env/type_info.cc @@ -14,13 +14,17 @@ #include "env/type_info.h" +#include #include +#include #include #include "absl/base/no_destructor.h" #include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "common/ast.h" #include "common/type.h" #include "common/type_kind.h" #include "env/config.h" @@ -180,5 +184,227 @@ absl::StatusOr TypeInfoToType( return DynType(); } } +absl::StatusOr TypeInfoToTypeSpec(const Config::TypeInfo& type_info) { + if (type_info.is_type_param) { + return TypeSpec(ParamTypeSpec(type_info.name)); + } + + std::optional type_kind = TypeNameToTypeKind(type_info.name); + if (!type_kind.has_value()) { + if (type_info.params.empty()) { + return TypeSpec(MessageTypeSpec(type_info.name)); + } else { + std::vector param_specs; + param_specs.reserve(type_info.params.size()); + for (const Config::TypeInfo& param : type_info.params) { + CEL_ASSIGN_OR_RETURN(TypeSpec param_spec, TypeInfoToTypeSpec(param)); + param_specs.push_back(std::move(param_spec)); + } + return TypeSpec(AbstractType(type_info.name, std::move(param_specs))); + } + } + + switch (*type_kind) { + case TypeKind::kNull: + return TypeSpec(NullTypeSpec()); + case TypeKind::kBool: + return TypeSpec(PrimitiveType::kBool); + case TypeKind::kInt: + return TypeSpec(PrimitiveType::kInt64); + case TypeKind::kUint: + return TypeSpec(PrimitiveType::kUint64); + case TypeKind::kDouble: + return TypeSpec(PrimitiveType::kDouble); + case TypeKind::kString: + return TypeSpec(PrimitiveType::kString); + case TypeKind::kBytes: + return TypeSpec(PrimitiveType::kBytes); + case TypeKind::kTimestamp: + return TypeSpec(WellKnownTypeSpec::kTimestamp); + case TypeKind::kDuration: + return TypeSpec(WellKnownTypeSpec::kDuration); + case TypeKind::kList: { + if (!type_info.params.empty()) { + CEL_ASSIGN_OR_RETURN(TypeSpec elem_type, + TypeInfoToTypeSpec(type_info.params[0])); + return TypeSpec( + ListTypeSpec(std::make_unique(std::move(elem_type)))); + } else { + return TypeSpec(ListTypeSpec()); + } + } + case TypeKind::kMap: { + if (type_info.params.empty()) { + return TypeSpec(MapTypeSpec()); + } + CEL_ASSIGN_OR_RETURN(TypeSpec key_type, + TypeInfoToTypeSpec(type_info.params[0])); + if (type_info.params.size() > 1) { + CEL_ASSIGN_OR_RETURN(TypeSpec value_type, + TypeInfoToTypeSpec(type_info.params[1])); + return TypeSpec( + MapTypeSpec(std::make_unique(std::move(key_type)), + std::make_unique(std::move(value_type)))); + } + return TypeSpec(MapTypeSpec( + std::make_unique(std::move(key_type)), nullptr)); + } + case TypeKind::kDyn: + return TypeSpec(DynTypeSpec()); + case TypeKind::kAny: + return TypeSpec(WellKnownTypeSpec::kAny); + case TypeKind::kBoolWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool)); + case TypeKind::kIntWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kInt64)); + case TypeKind::kUintWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kUint64)); + case TypeKind::kDoubleWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)); + case TypeKind::kStringWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kString)); + case TypeKind::kBytesWrapper: + return TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBytes)); + case TypeKind::kType: { + if (type_info.params.empty()) { + return TypeSpec(std::make_unique(DynTypeSpec())); + } + CEL_ASSIGN_OR_RETURN(TypeSpec type_param, + TypeInfoToTypeSpec(type_info.params[0])); + return TypeSpec(std::make_unique(std::move(type_param))); + } + default: + return TypeSpec(DynTypeSpec()); + } +} + +absl::StatusOr TypeSpecToTypeInfo(const TypeSpec& type_spec) { + Config::TypeInfo type_info; + + if (type_spec.has_dyn()) { + type_info.name = "dyn"; + } else if (type_spec.has_null()) { + type_info.name = "null"; + } else if (type_spec.has_primitive()) { + switch (type_spec.primitive()) { + case PrimitiveType::kBool: + type_info.name = "bool"; + break; + case PrimitiveType::kInt64: + type_info.name = "int"; + break; + case PrimitiveType::kUint64: + type_info.name = "uint"; + break; + case PrimitiveType::kDouble: + type_info.name = "double"; + break; + case PrimitiveType::kString: + type_info.name = "string"; + break; + case PrimitiveType::kBytes: + type_info.name = "bytes"; + break; + default: + return absl::InvalidArgumentError("Unspecified primitive type"); + } + } else if (type_spec.has_wrapper()) { + switch (type_spec.wrapper()) { + case PrimitiveType::kBool: + type_info.name = "bool_wrapper"; + break; + case PrimitiveType::kInt64: + type_info.name = "int_wrapper"; + break; + case PrimitiveType::kUint64: + type_info.name = "uint_wrapper"; + break; + case PrimitiveType::kDouble: + type_info.name = "double_wrapper"; + break; + case PrimitiveType::kString: + type_info.name = "string_wrapper"; + break; + case PrimitiveType::kBytes: + type_info.name = "bytes_wrapper"; + break; + default: + return absl::InvalidArgumentError("Unspecified wrapper type"); + } + } else if (type_spec.has_well_known()) { + switch (type_spec.well_known()) { + case WellKnownTypeSpec::kAny: + type_info.name = "any"; + break; + case WellKnownTypeSpec::kTimestamp: + type_info.name = "timestamp"; + break; + case WellKnownTypeSpec::kDuration: + type_info.name = "duration"; + break; + default: + return absl::InvalidArgumentError("Unspecified well known type"); + } + } else if (type_spec.has_list_type()) { + type_info.name = "list"; + const ListTypeSpec& list_type = type_spec.list_type(); + if (list_type.has_elem_type() && list_type.elem_type().is_specified()) { + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param, + TypeSpecToTypeInfo(list_type.elem_type())); + type_info.params.push_back(std::move(param)); + } + } else if (type_spec.has_map_type()) { + type_info.name = "map"; + const MapTypeSpec& map_type = type_spec.map_type(); + bool has_key = + map_type.has_key_type() && map_type.key_type().is_specified(); + bool has_value = + map_type.has_value_type() && map_type.value_type().is_specified(); + if (has_key || has_value) { + if (has_key) { + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param, + TypeSpecToTypeInfo(map_type.key_type())); + type_info.params.push_back(std::move(param)); + } else { + type_info.params.push_back(Config::TypeInfo{.name = "dyn"}); + } + if (has_value) { + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param_value, + TypeSpecToTypeInfo(map_type.value_type())); + type_info.params.push_back(std::move(param_value)); + } else { + type_info.params.push_back(Config::TypeInfo{.name = "dyn"}); + } + } + } else if (type_spec.has_message_type()) { + type_info.name = type_spec.message_type().type(); + } else if (type_spec.has_type_param()) { + type_info.name = type_spec.type_param().type(); + type_info.is_type_param = true; + } else if (type_spec.has_type()) { + type_info.name = "type"; + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param, + TypeSpecToTypeInfo(type_spec.type())); + type_info.params.push_back(std::move(param)); + } else if (type_spec.has_abstract_type()) { + type_info.name = type_spec.abstract_type().name(); + for (const TypeSpec& param_spec : + type_spec.abstract_type().parameter_types()) { + CEL_ASSIGN_OR_RETURN(Config::TypeInfo param, + TypeSpecToTypeInfo(param_spec)); + type_info.params.push_back(std::move(param)); + } + } else if (type_spec.has_error()) { + return absl::InvalidArgumentError( + "ErrorType cannot be converted to TypeInfo"); + } else if (type_spec.has_function()) { + return absl::InvalidArgumentError( + "FunctionType cannot be converted to TypeInfo"); + } else { + return absl::InvalidArgumentError("Unknown TypeSpec kind"); + } + + return type_info; +} } // namespace cel diff --git a/env/type_info.h b/env/type_info.h index bb3cfde43..3f802ce1a 100644 --- a/env/type_info.h +++ b/env/type_info.h @@ -16,6 +16,7 @@ #define THIRD_PARTY_CEL_CPP_ENV_TYPE_INFO_H_ #include "absl/status/statusor.h" +#include "common/ast.h" #include "common/type.h" #include "env/config.h" #include "google/protobuf/arena.h" @@ -30,6 +31,12 @@ absl::StatusOr TypeInfoToType( const Config::TypeInfo& type_info, const google::protobuf::DescriptorPool* descriptor_pool, google::protobuf::Arena* arena); +// Converts a Config::TypeInfo to a cel::TypeSpec. +absl::StatusOr TypeInfoToTypeSpec(const Config::TypeInfo& type_info); + +// Converts a cel::TypeSpec to a Config::TypeInfo. +absl::StatusOr TypeSpecToTypeInfo(const TypeSpec& type_spec); + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_ENV_TYPE_INFO_H_ diff --git a/env/type_info_test.cc b/env/type_info_test.cc index 015d8a928..f9d46f9a9 100644 --- a/env/type_info_test.cc +++ b/env/type_info_test.cc @@ -14,9 +14,14 @@ #include "env/type_info.h" +#include +#include +#include #include #include +#include "absl/status/status.h" +#include "common/ast/metadata.h" #include "common/type.h" #include "common/type_proto.h" #include "env/config.h" @@ -28,9 +33,27 @@ #include "google/protobuf/text_format.h" namespace cel { + +std::ostream& operator<<(std::ostream& os, const Config::TypeInfo& type_info) { + if (type_info.is_type_param) { + os << "?"; + } + os << type_info.name; + if (!type_info.params.empty()) { + os << "<"; + for (size_t i = 0; i < type_info.params.size(); ++i) { + if (i > 0) os << ", "; + os << type_info.params[i]; + } + os << ">"; + } + return os; +} + namespace { using absl_testing::IsOk; +using absl_testing::StatusIs; using testing::ValuesIn; struct TestCase { @@ -127,5 +150,151 @@ std::vector GetTestCases() { INSTANTIATE_TEST_SUITE_P(TypeInfoTest, TypeInfoTest, ValuesIn(GetTestCases())); +bool TypeInfoEqImpl(const Config::TypeInfo& actual, + const Config::TypeInfo& expected) { + if (actual.name != expected.name) return false; + if (actual.is_type_param != expected.is_type_param) return false; + if (actual.params.size() != expected.params.size()) return false; + for (size_t i = 0; i < actual.params.size(); ++i) { + if (!TypeInfoEqImpl(actual.params[i], expected.params[i])) return false; + } + return true; +} + +MATCHER_P(TypeInfoEq, expected, "") { return TypeInfoEqImpl(arg, expected); } + +struct TypeSpecTestCase { + TypeSpec type_spec; + Config::TypeInfo expected_type_info; +}; + +using TypeSpecToTypeInfoTest = testing::TestWithParam; + +TEST_P(TypeSpecToTypeInfoTest, Convert) { + const TypeSpecTestCase& param = GetParam(); + ASSERT_OK_AND_ASSIGN(Config::TypeInfo actual_type_info, + TypeSpecToTypeInfo(param.type_spec)); + EXPECT_THAT(actual_type_info, TypeInfoEq(param.expected_type_info)); +} + +std::vector GetTypeSpecTestCases() { + return { + TypeSpecTestCase{ + .type_spec = TypeSpec(PrimitiveType::kInt64), + .expected_type_info = {.name = "int"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + ListTypeSpec(std::make_unique(PrimitiveType::kInt64))), + .expected_type_info = {.name = "list", + .params = {Config::TypeInfo{.name = "int"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(ListTypeSpec()), + .expected_type_info = {.name = "list"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kString), + std::make_unique(PrimitiveType::kInt64))), + .expected_type_info = {.name = "map", + .params = {Config::TypeInfo{.name = "string"}, + Config::TypeInfo{.name = "int"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(MapTypeSpec()), + .expected_type_info = {.name = "map"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + MessageTypeSpec("cel.expr.conformance.proto2.TestAllTypes")), + .expected_type_info = + {.name = "cel.expr.conformance.proto2.TestAllTypes"}, + }, + TypeSpecTestCase{ + .type_spec = + TypeSpec(AbstractType("A", {TypeSpec(ParamTypeSpec("B"))})), + .expected_type_info = {.name = "A", + .params = {Config::TypeInfo{ + .name = "B", .is_type_param = true}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(WellKnownTypeSpec::kAny), + .expected_type_info = {.name = "any"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(WellKnownTypeSpec::kTimestamp), + .expected_type_info = {.name = "timestamp"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kDouble)), + .expected_type_info = {.name = "double_wrapper"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + std::make_unique(WellKnownTypeSpec::kDuration)), + .expected_type_info = {.name = "type", + .params = {Config::TypeInfo{.name = + "duration"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(std::make_unique(DynTypeSpec())), + .expected_type_info = {.name = "type", + .params = {Config::TypeInfo{.name = "dyn"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(DynTypeSpec{}), + .expected_type_info = {.name = "dyn"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec(NullTypeSpec{}), + .expected_type_info = {.name = "null"}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kString), + std::make_unique(DynTypeSpec()))), + .expected_type_info = {.name = "map", + .params = {Config::TypeInfo{.name = "string"}, + Config::TypeInfo{.name = "dyn"}}}, + }, + TypeSpecTestCase{ + .type_spec = TypeSpec( + MapTypeSpec(std::make_unique(DynTypeSpec()), + std::make_unique(PrimitiveType::kInt64))), + .expected_type_info = {.name = "map", + .params = {Config::TypeInfo{.name = "dyn"}, + Config::TypeInfo{.name = "int"}}}, + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(TypeSpecToTypeInfoTest, TypeSpecToTypeInfoTest, + ValuesIn(GetTypeSpecTestCases())); + +using TypeInfoToTypeSpecTest = testing::TestWithParam; + +TEST_P(TypeInfoToTypeSpecTest, Convert) { + const TypeSpecTestCase& param = GetParam(); + ASSERT_OK_AND_ASSIGN(TypeSpec actual_type_spec, + TypeInfoToTypeSpec(param.expected_type_info)); + EXPECT_EQ(actual_type_spec, param.type_spec); +} + +INSTANTIATE_TEST_SUITE_P(TypeInfoToTypeSpecTest, TypeInfoToTypeSpecTest, + ValuesIn(GetTypeSpecTestCases())); + +TEST(TypeSpecToTypeInfoTest, ErrorConversions) { + EXPECT_THAT(TypeSpecToTypeInfo(TypeSpec(ErrorTypeSpec::kValue)), + StatusIs(absl::StatusCode::kInvalidArgument, + "ErrorType cannot be converted to TypeInfo")); + EXPECT_THAT(TypeSpecToTypeInfo(TypeSpec(FunctionTypeSpec())), + StatusIs(absl::StatusCode::kInvalidArgument, + "FunctionType cannot be converted to TypeInfo")); + EXPECT_THAT( + TypeSpecToTypeInfo(TypeSpec(UnsetTypeSpec())), + StatusIs(absl::StatusCode::kInvalidArgument, "Unknown TypeSpec kind")); +} + } // namespace } // namespace cel From a8b3224f86712c8c564b9a54abf4eac7cc5ee39a Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 9 Jun 2026 11:04:51 -0700 Subject: [PATCH 057/350] Add the function AddContextDeclarationWithProtoTypeMask to the type checker. PiperOrigin-RevId: 929295083 --- checker/internal/BUILD | 6 + checker/internal/type_check_env.cc | 5 + checker/internal/type_check_env.h | 14 + checker/internal/type_checker_builder_impl.cc | 56 +++- checker/internal/type_checker_builder_impl.h | 9 + .../type_checker_builder_impl_test.cc | 311 ++++++++++++++++++ checker/type_checker_builder.h | 22 ++ checker/type_checker_builder_factory_test.cc | 47 +++ 8 files changed, 464 insertions(+), 6 deletions(-) diff --git a/checker/internal/BUILD b/checker/internal/BUILD index 26c7b543f..777457830 100644 --- a/checker/internal/BUILD +++ b/checker/internal/BUILD @@ -66,6 +66,8 @@ cc_library( hdrs = ["type_check_env.h"], deps = [ ":descriptor_pool_type_introspector", + ":proto_type_mask", + ":proto_type_mask_registry", "//common:constant", "//common:container", "//common:decl", @@ -76,6 +78,7 @@ cc_library( "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", @@ -129,6 +132,7 @@ cc_library( deps = [ ":format_type_name", ":namespace_generator", + ":proto_type_mask", ":type_check_env", ":type_inference_context", "//checker:checker_options", @@ -154,6 +158,7 @@ cc_library( "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:btree", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", @@ -226,6 +231,7 @@ cc_test( "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", diff --git a/checker/internal/type_check_env.cc b/checker/internal/type_check_env.cc index 763d9ba46..47487220c 100644 --- a/checker/internal/type_check_env.cc +++ b/checker/internal/type_check_env.cc @@ -16,6 +16,7 @@ #include #include +#include #include "absl/base/nullability.h" #include "absl/status/statusor.h" @@ -96,6 +97,10 @@ absl::StatusOr> TypeCheckEnv::LookupTypeConstant( absl::StatusOr> TypeCheckEnv::LookupStructField( absl::string_view type_name, absl::string_view field_name) const { + if (proto_type_mask_registry_ != nullptr && + !proto_type_mask_registry_->FieldIsVisible(type_name, field_name)) { + return absl::nullopt; + } // Check the type providers in registration order. // Note: this doesn't allow for shadowing a type with a subset type of the // same name -- the later type provider will still be considered when diff --git a/checker/internal/type_check_env.h b/checker/internal/type_check_env.h index 15f8ecc4d..00fea0ba3 100644 --- a/checker/internal/type_check_env.h +++ b/checker/internal/type_check_env.h @@ -25,16 +25,20 @@ #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/memory/memory.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" #include "checker/internal/descriptor_pool_type_introspector.h" +#include "checker/internal/proto_type_mask.h" +#include "checker/internal/proto_type_mask_registry.h" #include "common/constant.h" #include "common/container.h" #include "common/decl.h" #include "common/type.h" #include "common/type_introspector.h" +#include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -154,6 +158,14 @@ class TypeCheckEnv { variables_[decl.name()] = std::move(decl); } + absl::Status CreateProtoTypeMaskRegistry( + const std::vector& proto_type_masks) { + CEL_ASSIGN_OR_RETURN(proto_type_mask_registry_, + ProtoTypeMaskRegistry::Create(descriptor_pool_.get(), + proto_type_masks)); + return absl::OkStatus(); + } + const absl::flat_hash_map& functions() const { return functions_; } @@ -224,6 +236,8 @@ class TypeCheckEnv { absl::flat_hash_map variables_; absl::flat_hash_map functions_; + std::shared_ptr proto_type_mask_registry_; + // Type providers for custom types. std::vector> type_providers_; diff --git a/checker/internal/type_checker_builder_impl.cc b/checker/internal/type_checker_builder_impl.cc index 85b581e83..9b91fc926 100644 --- a/checker/internal/type_checker_builder_impl.cc +++ b/checker/internal/type_checker_builder_impl.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -23,13 +24,16 @@ #include "absl/base/no_destructor.h" #include "absl/base/nullability.h" #include "absl/cleanup/cleanup.h" +#include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "checker/internal/proto_type_mask.h" #include "checker/internal/type_check_env.h" #include "checker/internal/type_checker_impl.h" #include "checker/type_checker.h" @@ -86,10 +90,19 @@ absl::Status CheckStdMacroOverlap(const FunctionDecl& decl) { } absl::Status AddWellKnownContextDeclarationVariables( - const google::protobuf::Descriptor* absl_nonnull descriptor, TypeCheckEnv& env, - bool use_json_name) { + const google::protobuf::Descriptor* absl_nonnull descriptor, + const absl::flat_hash_map>& + context_type_fields, + TypeCheckEnv& env, bool use_json_name) { for (int i = 0; i < descriptor->field_count(); ++i) { const google::protobuf::FieldDescriptor* field = descriptor->field(i); + // Skip fields that are hidden because of a proto type mask. + auto map_iterator = context_type_fields.find(descriptor->full_name()); + if (map_iterator != context_type_fields.end() && + !map_iterator->second.contains(field->name())) { + continue; + } Type type = MessageTypeField(field).GetType(); if (type.IsEnum()) { type = IntType(); @@ -109,11 +122,15 @@ absl::Status AddWellKnownContextDeclarationVariables( } absl::Status AddContextDeclarationVariables( - const google::protobuf::Descriptor* absl_nonnull descriptor, TypeCheckEnv& env) { + const google::protobuf::Descriptor* absl_nonnull descriptor, + const absl::flat_hash_map>& + context_type_fields, + TypeCheckEnv& env) { const bool use_json_name = env.proto_type_introspector().use_json_name(); if (IsWellKnownMessageType(descriptor)) { - return AddWellKnownContextDeclarationVariables(descriptor, env, - use_json_name); + return AddWellKnownContextDeclarationVariables( + descriptor, context_type_fields, env, use_json_name); } CEL_ASSIGN_OR_RETURN(auto fields, env.proto_type_introspector().ListFieldsForStructType( @@ -131,6 +148,13 @@ absl::Status AddContextDeclarationVariables( absl::string_view name = field_entry.name; + // Skip fields that are hidden because of a proto type mask. + auto map_iterator = context_type_fields.find(descriptor->full_name()); + if (map_iterator != context_type_fields.end() && + !map_iterator->second.contains(name)) { + continue; + } + if (!env.InsertVariableIfAbsent(MakeVariableDecl(name, type))) { return absl::AlreadyExistsError( absl::StrCat("variable '", name, @@ -317,7 +341,8 @@ absl::Status TypeCheckerBuilderImpl::ApplyConfig( } for (const google::protobuf::Descriptor* context_type : config.context_types) { - CEL_RETURN_IF_ERROR(AddContextDeclarationVariables(context_type, env)); + CEL_RETURN_IF_ERROR(AddContextDeclarationVariables( + context_type, config.context_type_fields, env)); } for (VariableDeclRecord& var : config.variables) { @@ -339,6 +364,8 @@ absl::Status TypeCheckerBuilderImpl::ApplyConfig( } } + CEL_RETURN_IF_ERROR(env.CreateProtoTypeMaskRegistry(config.proto_type_masks)); + return absl::OkStatus(); } @@ -462,6 +489,23 @@ absl::Status TypeCheckerBuilderImpl::AddContextDeclaration( return absl::OkStatus(); } +absl::Status TypeCheckerBuilderImpl::AddContextDeclarationWithProtoTypeMask( + absl::string_view type, std::vector field_paths) { + if (field_paths.empty()) { + return absl::InvalidArgumentError("field paths cannot be the empty set"); + } + + ProtoTypeMask proto_type_mask(std::string(type), field_paths); + target_config_->proto_type_masks.push_back(proto_type_mask); + + CEL_RETURN_IF_ERROR(AddContextDeclaration(type)); + CEL_ASSIGN_OR_RETURN( + absl::btree_set field_names, + proto_type_mask.GetFieldNames(template_env_.descriptor_pool())); + target_config_->context_type_fields.insert({type, std::move(field_names)}); + return absl::OkStatus(); +} + absl::Status TypeCheckerBuilderImpl::AddFunction(const FunctionDecl& decl) { CEL_RETURN_IF_ERROR( ValidateFunctionDecl(decl, options_.enable_type_parameter_name_validation, diff --git a/checker/internal/type_checker_builder_impl.h b/checker/internal/type_checker_builder_impl.h index 646a5d16f..9895a8aee 100644 --- a/checker/internal/type_checker_builder_impl.h +++ b/checker/internal/type_checker_builder_impl.h @@ -21,6 +21,7 @@ #include #include "absl/base/nullability.h" +#include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" @@ -28,6 +29,7 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "checker/checker_options.h" +#include "checker/internal/proto_type_mask.h" #include "checker/internal/type_check_env.h" #include "checker/type_checker.h" #include "checker/type_checker_builder.h" @@ -76,6 +78,8 @@ class TypeCheckerBuilderImpl : public TypeCheckerBuilder { absl::Status AddVariable(const VariableDecl& decl) override; absl::Status AddOrReplaceVariable(const VariableDecl& decl) override; absl::Status AddContextDeclaration(absl::string_view type) override; + absl::Status AddContextDeclarationWithProtoTypeMask( + absl::string_view type, std::vector field_paths) override; absl::Status AddFunction(const FunctionDecl& decl) override; absl::Status MergeFunction(const FunctionDecl& decl) override; @@ -130,6 +134,11 @@ class TypeCheckerBuilderImpl : public TypeCheckerBuilder { std::vector functions; std::vector> type_providers; std::vector context_types; + // Maps context type names to fields names to add as variables. + // Only includes context types that are defined with proto type masks. + absl::flat_hash_map> + context_type_fields; + std::vector proto_type_masks; }; absl::Status BuildLibraryConfig(const CheckerLibrary& library, diff --git a/checker/internal/type_checker_builder_impl_test.cc b/checker/internal/type_checker_builder_impl_test.cc index 494e7e440..913e704ee 100644 --- a/checker/internal/type_checker_builder_impl_test.cc +++ b/checker/internal/type_checker_builder_impl_test.cc @@ -15,12 +15,15 @@ #include "checker/internal/type_checker_builder_impl.h" #include +#include #include #include +#include #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "checker/checker_options.h" @@ -107,6 +110,168 @@ INSTANTIATE_TEST_SUITE_P( MapTypeSpec(std::make_unique(PrimitiveType::kString), std::make_unique(DynTypeSpec())))})); +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnEmptyFieldPaths) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {}), + StatusIs(absl::StatusCode::kInvalidArgument, + "field paths cannot be the empty set")); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnUnknownFieldPath) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT( + builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"unknown_field"}), + StatusIs(absl::StatusCode::kInvalidArgument, + "could not select field 'unknown_field' from type " + "'cel.expr.conformance.proto3.TestAllTypes'")); +} + +class ContextDeclsWithProtoTypeMaskFieldsDefinedTest + : public testing::TestWithParam {}; + +std::string LogFieldName(absl::string_view field_name, absl::string_view expr) { + return absl::StrCat("field_name: ", field_name, ", expr: ", expr); +} + +TEST_P(ContextDeclsWithProtoTypeMaskFieldsDefinedTest, + ContextDeclsWithProtoTypeMaskFieldsDefined) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT( + builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {GetParam().expr}), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder.Build()); + std::vector field_names = { + "single_int64", "single_uint32", "single_double", + "single_string", "single_any", "single_duration", + "single_bool_wrapper", "list_value", "standalone_message", + "standalone_enum", "repeated_bytes", "repeated_nested_message", + "map_int32_timestamp", "single_struct"}; + for (auto& field_name : field_names) { + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst(field_name)); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + if (field_name == GetParam().expr) { + // The field name that is part of the proto type mask is visible. + ASSERT_TRUE(result.IsValid()) + << LogFieldName(field_name, GetParam().expr); + EXPECT_EQ(result.GetAst()->GetReturnType(), GetParam().expected_type) + << LogFieldName(field_name, GetParam().expr); + } else { + // The field names that are not part of the proto type mask are not + // visible. + EXPECT_FALSE(result.IsValid()) + << LogFieldName(field_name, GetParam().expr); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + TestAllTypes, ContextDeclsWithProtoTypeMaskFieldsDefinedTest, + testing::Values( + ContextDeclsTestCase{"single_int64", TypeSpec(PrimitiveType::kInt64)}, + ContextDeclsTestCase{"single_uint32", TypeSpec(PrimitiveType::kUint64)}, + ContextDeclsTestCase{"single_double", TypeSpec(PrimitiveType::kDouble)}, + ContextDeclsTestCase{"single_string", TypeSpec(PrimitiveType::kString)}, + ContextDeclsTestCase{"single_any", TypeSpec(WellKnownTypeSpec::kAny)}, + ContextDeclsTestCase{"single_duration", + TypeSpec(WellKnownTypeSpec::kDuration)}, + ContextDeclsTestCase{ + "single_bool_wrapper", + TypeSpec(PrimitiveTypeWrapper(PrimitiveType::kBool))}, + ContextDeclsTestCase{ + "list_value", + TypeSpec(ListTypeSpec(std::make_unique(DynTypeSpec())))}, + ContextDeclsTestCase{ + "standalone_message", + TypeSpec(MessageTypeSpec( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage"))}, + ContextDeclsTestCase{"standalone_enum", + TypeSpec(PrimitiveType::kInt64)}, + ContextDeclsTestCase{"repeated_bytes", + TypeSpec(ListTypeSpec(std::make_unique( + PrimitiveType::kBytes)))}, + ContextDeclsTestCase{ + "repeated_nested_message", + TypeSpec(ListTypeSpec(std::make_unique(MessageTypeSpec( + "cel.expr.conformance.proto3.TestAllTypes.NestedMessage"))))}, + ContextDeclsTestCase{ + "map_int32_timestamp", + TypeSpec(MapTypeSpec( + std::make_unique(PrimitiveType::kInt64), + std::make_unique(WellKnownTypeSpec::kTimestamp)))}, + ContextDeclsTestCase{ + "single_struct", + TypeSpec( + MapTypeSpec(std::make_unique(PrimitiveType::kString), + std::make_unique(DynTypeSpec())))})); + +TEST(ContextDeclsWithProtoTypeMaskTest, FieldsInMaskAreVisibleFieldAccess) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb", "payload.single_int32"}), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder.Build()); + // Visible field: standalone_message.bb + ASSERT_OK_AND_ASSIGN(auto ast, + MakeTestParsedAst("payload.standalone_message.bb")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + EXPECT_EQ(result.GetAst()->GetReturnType(), TypeSpec(PrimitiveType::kInt64)); + // Visible field: single_int32 + ASSERT_OK_AND_ASSIGN(ast, MakeTestParsedAst("payload.single_int32")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + EXPECT_EQ(result.GetAst()->GetReturnType(), TypeSpec(PrimitiveType::kInt64)); + // Not Visible field: single_int64 + ASSERT_OK_AND_ASSIGN(ast, MakeTestParsedAst("payload.single_int64")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_FALSE(result.IsValid()); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, FieldsInMaskAreVisibleFieldAssignment) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.NestedTestAllTypes", + {"payload.standalone_message.bb", "payload.single_int32"}), + IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder.Build()); + // Visible field: standalone_message.bb + ASSERT_OK_AND_ASSIGN( + auto ast, + MakeTestParsedAst( + R"(cel.expr.conformance.proto3.TestAllTypes.NestedMessage{bb: 12345})")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + // Visible field: single_int32 + ASSERT_OK_AND_ASSIGN( + ast, + MakeTestParsedAst( + R"(cel.expr.conformance.proto3.TestAllTypes{single_int32: 12345})")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + // Not Visible field: single_int64 + ASSERT_OK_AND_ASSIGN( + ast, + MakeTestParsedAst( + R"(cel.expr.conformance.proto3.TestAllTypes{single_int64: 12345})")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_FALSE(result.IsValid()); +} + TEST(ContextDeclsTest, ErrorOnDuplicateContextDeclaration) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -120,6 +285,20 @@ TEST(ContextDeclsTest, ErrorOnDuplicateContextDeclaration) { "already exists")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnDuplicateContextDeclaration) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT( + builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"standalone_message"}), + IsOk()); + EXPECT_THAT( + builder.AddContextDeclaration("cel.expr.conformance.proto3.TestAllTypes"), + StatusIs(absl::StatusCode::kAlreadyExists, + "context declaration 'cel.expr.conformance.proto3.TestAllTypes' " + "already exists")); +} + TEST(ContextDeclsTest, ErrorOnContextDeclarationNotFound) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -129,6 +308,16 @@ TEST(ContextDeclsTest, ErrorOnContextDeclarationNotFound) { "context declaration 'com.example.UnknownType' not found")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnContextDeclarationNotFound) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + EXPECT_THAT( + builder.AddContextDeclarationWithProtoTypeMask("com.example.UnknownType", + {"any_field_name"}), + StatusIs(absl::StatusCode::kNotFound, + "context declaration 'com.example.UnknownType' not found")); +} + TEST(ContextDeclsTest, ErrorOnNonStructMessageType) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -139,6 +328,17 @@ TEST(ContextDeclsTest, ErrorOnNonStructMessageType) { "context declaration 'google.protobuf.Timestamp' is not a struct")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnNonStructMessageType) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + EXPECT_THAT( + builder.AddContextDeclarationWithProtoTypeMask( + "google.protobuf.Timestamp", {"any_field_name"}), + StatusIs( + absl::StatusCode::kInvalidArgument, + "context declaration 'google.protobuf.Timestamp' is not a struct")); +} + TEST(ContextDeclsTest, CustomStructNotSupported) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -160,6 +360,28 @@ TEST(ContextDeclsTest, CustomStructNotSupported) { "context declaration 'com.example.MyStruct' not found")); } +TEST(ContextDeclsWithProtoTypeMaskTest, CustomStructNotSupported) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + class MyTypeProvider : public cel::TypeIntrospector { + public: + absl::StatusOr> FindTypeImpl( + absl::string_view name) const override { + if (name == "com.example.MyStruct") { + return common_internal::MakeBasicStructType("com.example.MyStruct"); + } + return absl::nullopt; + } + }; + + builder.AddTypeProvider(std::make_unique()); + + EXPECT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "com.example.MyStruct", {"any_field_name"}), + StatusIs(absl::StatusCode::kNotFound, + "context declaration 'com.example.MyStruct' not found")); +} + TEST(ContextDeclsTest, ErrorOnOverlappingContextDeclaration) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -179,6 +401,69 @@ TEST(ContextDeclsTest, ErrorOnOverlappingContextDeclaration) { "declaration: 'cel.expr.conformance.proto2.TestAllTypes')")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnOverlappingContextDeclaration) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT( + builder.AddContextDeclaration("cel.expr.conformance.proto3.TestAllTypes"), + IsOk()); + // We resolve the context declaration variables at the Build() call, so the + // error surfaces then. + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto2.TestAllTypes", {"single_int32"}), + IsOk()); + + EXPECT_THAT( + builder.Build(), + StatusIs(absl::StatusCode::kAlreadyExists, + "variable 'single_int32' declared multiple times (from context " + "declaration: 'cel.expr.conformance.proto2.TestAllTypes')")); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, + ErrorOnOverlappingContextDeclarationBothProtoTypeMasks) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int32"}), + IsOk()); + // We resolve the context declaration variables at the Build() call, so the + // error surfaces then. + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto2.TestAllTypes", {"single_int32"}), + IsOk()); + + EXPECT_THAT( + builder.Build(), + StatusIs(absl::StatusCode::kAlreadyExists, + "variable 'single_int32' declared multiple times (from context " + "declaration: 'cel.expr.conformance.proto2.TestAllTypes')")); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, + NonOverlappingContextDeclarationBothProtoTypeMasks) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int32"}), + IsOk()); + // We resolve the context declaration variables at the Build() call, so the + // error surfaces then. + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto2.NestedTestAllTypes", + {"payload.single_int64"}), + IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder.Build()); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("single_int32")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(ast, MakeTestParsedAst("payload.single_int64")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); +} + TEST(ContextDeclsTest, ErrorOnOverlappingVariableDeclaration) { TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), {}); @@ -193,6 +478,32 @@ TEST(ContextDeclsTest, ErrorOnOverlappingVariableDeclaration) { "variable 'single_int64' declared multiple times")); } +TEST(ContextDeclsWithProtoTypeMaskTest, ErrorOnOverlappingVariableDeclaration) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int64"}), + IsOk()); + ASSERT_THAT(builder.AddVariable(MakeVariableDecl("single_int64", IntType())), + IsOk()); + + EXPECT_THAT(builder.Build(), + StatusIs(absl::StatusCode::kAlreadyExists, + "variable 'single_int64' declared multiple times")); +} + +TEST(ContextDeclsWithProtoTypeMaskTest, NonOverlappingVariableDeclaration) { + TypeCheckerBuilderImpl builder(internal::GetSharedTestingDescriptorPool(), + {}); + ASSERT_THAT(builder.AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int32"}), + IsOk()); + ASSERT_THAT(builder.AddVariable(MakeVariableDecl("single_int64", IntType())), + IsOk()); + + EXPECT_THAT(builder.Build(), IsOk()); +} + TEST(TypeCheckerBuilderImplTest, InvalidTypeParamNameVariableValidationDisabled) { CheckerOptions options; diff --git a/checker/type_checker_builder.h b/checker/type_checker_builder.h index 5dd1f5256..f145b8a98 100644 --- a/checker/type_checker_builder.h +++ b/checker/type_checker_builder.h @@ -17,6 +17,7 @@ #include #include +#include #include "absl/base/nullability.h" #include "absl/functional/any_invocable.h" @@ -102,6 +103,27 @@ class TypeCheckerBuilder { // Note: only protobuf backed struct types are supported at this time. virtual absl::Status AddContextDeclaration(absl::string_view type) = 0; + // Declares struct type by fully qualified name as a context declaration. + // + // This version accepts a mask in terms of field selections from the + // context type. The mask specifies which fields are visible on the + // struct and its members. The visible fields for a type accumulate + // across calls. This is a lightweight way to adjust the type checking + // behavior for a group of related types. + // + // Context declarations are a way to declare a group of variables based on the + // definition of a struct type. Each top level field of the struct that is + // also the first field name in a field path is declared as an individual + // variable of the field type. + // + // It is an error if the type contains a field that overlaps with another + // declared variable. It is an error if the input field paths is the empty + // set. + // + // Note: only protobuf backed struct types are supported at this time. + virtual absl::Status AddContextDeclarationWithProtoTypeMask( + absl::string_view type, std::vector field_paths) = 0; + // Adds a function declaration that may be referenced in expressions checked // with the resulting TypeChecker. virtual absl::Status AddFunction(const FunctionDecl& decl) = 0; diff --git a/checker/type_checker_builder_factory_test.cc b/checker/type_checker_builder_factory_test.cc index 38430de5f..9c4775e7f 100644 --- a/checker/type_checker_builder_factory_test.cc +++ b/checker/type_checker_builder_factory_test.cc @@ -396,6 +396,27 @@ TEST(TypeCheckerBuilderTest, AddContextDeclaration) { EXPECT_TRUE(result.IsValid()); } +TEST(TypeCheckerBuilderTest, AddContextDeclarationWithProtoTypeMask) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTypeCheckerBuilder(GetSharedTestingDescriptorPool())); + + ASSERT_OK_AND_ASSIGN( + auto fn_decl, + MakeFunctionDecl("increment", MakeOverloadDecl("increment_int", IntType(), + IntType()))); + + ASSERT_THAT(builder->AddContextDeclarationWithProtoTypeMask( + "cel.expr.conformance.proto3.TestAllTypes", {"single_int64"}), + IsOk()); + ASSERT_THAT(builder->AddFunction(fn_decl), IsOk()); + + ASSERT_OK_AND_ASSIGN(auto checker, builder->Build()); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("increment(single_int64)")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, checker->Check(std::move(ast))); + EXPECT_TRUE(result.IsValid()); +} + TEST(TypeCheckerBuilderTest, WellKnownTypeContextDeclarationError) { ASSERT_OK_AND_ASSIGN( std::unique_ptr builder, @@ -428,6 +449,32 @@ TEST(TypeCheckerBuilderTest, AllowWellKnownTypeContextDeclaration) { ASSERT_TRUE(result.IsValid()); } +TEST(TypeCheckerBuilderTest, + AllowWellKnownTypeContextDeclarationWithProtoTypeMask) { + CheckerOptions options; + options.allow_well_known_type_context_declarations = true; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTypeCheckerBuilder(GetSharedTestingDescriptorPool(), options)); + + ASSERT_THAT(builder->AddContextDeclarationWithProtoTypeMask( + "google.protobuf.Any", {"value"}), + IsOk()); + ASSERT_THAT(builder->AddLibrary(StandardCheckerLibrary()), IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr type_checker, + builder->Build()); + // Visible field: value + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("value")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, + type_checker->Check(std::move(ast))); + ASSERT_TRUE(result.IsValid()); + // Not visible field: type_url + ASSERT_OK_AND_ASSIGN(ast, MakeTestParsedAst("type_url")); + ASSERT_OK_AND_ASSIGN(result, type_checker->Check(std::move(ast))); + ASSERT_FALSE(result.IsValid()); +} + TEST(TypeCheckerBuilderTest, AllowWellKnownTypeContextDeclarationStruct) { CheckerOptions options; options.allow_well_known_type_context_declarations = true; From a721a9a8d287edd074cd7c4fd8157aef9fa3074a Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 9 Jun 2026 13:28:55 -0700 Subject: [PATCH 058/350] Add support for old style variable format in env yaml parser. PiperOrigin-RevId: 929372359 --- env/env_yaml.cc | 10 ++++++++-- env/env_yaml_test.cc | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/env/env_yaml.cc b/env/env_yaml.cc index 8c635e65f..1bbfe6b36 100644 --- a/env/env_yaml.cc +++ b/env/env_yaml.cc @@ -620,8 +620,14 @@ absl::Status ParseVariableConfigs(Config& config, absl::string_view yaml, } variable_config.description = GetString(yaml, description); } - - CEL_ASSIGN_OR_RETURN(auto type_info, ParseTypeInfo(variable, yaml)); + const YAML::Node type = variable["type"]; + Config::TypeInfo type_info; + if (type.IsDefined() && !type.IsScalar()) { + // Old format, type spec is in 'type' instead of directly embedded. + CEL_ASSIGN_OR_RETURN(type_info, ParseTypeInfo(variable["type"], yaml)); + } else { + CEL_ASSIGN_OR_RETURN(type_info, ParseTypeInfo(variable, yaml)); + } ConstantKindCase constant_kind_case = GetConstantKindCase(type_info.name); std::string value_str; YAML::Node value = variable["value"]; diff --git a/env/env_yaml_test.cc b/env/env_yaml_test.cc index f6bde59c9..a60048617 100644 --- a/env/env_yaml_test.cc +++ b/env/env_yaml_test.cc @@ -242,6 +242,24 @@ TEST(EnvYamlTest, ParseVariableConfigWithTypeParamsLegacySyntax) { EXPECT_THAT(type_info.params[1].params, IsEmpty()); } +TEST(EnvYamlTest, ParseVariableConfigWithNestedRuleOldFormat) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + variables: + - name: "x" + type: + type_name: "int" + )yaml")); + + ASSERT_THAT(config.GetVariableConfigs(), SizeIs(1)); + const Config::VariableConfig& variable_config = + config.GetVariableConfigs()[0]; + EXPECT_EQ(variable_config.name, "x"); + const auto& type_info = variable_config.type_info; + EXPECT_EQ(type_info.name, "int"); + EXPECT_FALSE(type_info.is_type_param); + EXPECT_THAT(type_info.params, IsEmpty()); +} + struct ParseConstantTestCase { std::string type; std::string value; From d301931cafb674699daa54d922691ccddc6c55ba Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 9 Jun 2026 16:19:27 -0700 Subject: [PATCH 059/350] internal PiperOrigin-RevId: 929463453 --- env/env.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/env/env.cc b/env/env.cc index 42652ce59..6cd3a3cdc 100644 --- a/env/env.cc +++ b/env/env.cc @@ -122,6 +122,7 @@ absl::StatusOr FunctionConfigToFunctionDecl( Env::Env() { compiler_options_.parser_options.enable_quoted_identifiers = true; + compiler_options_.adapt_parser_errors = true; } absl::StatusOr> Env::NewCompilerBuilder() { From 167e797bdd017b074c1f4fe4a80598d2fa1e1613 Mon Sep 17 00:00:00 2001 From: Antoine Pietri Date: Thu, 11 Jun 2026 00:28:02 -0700 Subject: [PATCH 060/350] Fix source range offsets for lists, maps and messages. Update the parser to derive the source range for list, map, and struct/message creation expressions from the full parser rule context rather than just the opening token. This ensures the recorded offsets in `EnrichedSourceInfo` span the entire composite expression. PiperOrigin-RevId: 930338742 --- parser/parser.cc | 6 +++--- parser/parser_test.cc | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/parser/parser.cc b/parser/parser.cc index 6c6434319..a858337a4 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -1104,7 +1104,7 @@ std::any ParserVisitor::visitCreateMessage( } else { name = absl::StrJoin(parts, "."); } - int64_t obj_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + int64_t obj_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); std::vector fields; if (ctx->entries) { fields = visitFields(ctx->entries); @@ -1206,7 +1206,7 @@ std::any ParserVisitor::visitNested(CelParser::NestedContext* ctx) { } std::any ParserVisitor::visitCreateList(CelParser::CreateListContext* ctx) { - int64_t list_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + int64_t list_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); auto elems = visitList(ctx->elems); return ExprToAny(factory_.NewList(list_id, std::move(elems))); } @@ -1244,7 +1244,7 @@ std::vector ParserVisitor::visitList(CelParser::ExprListContext* ctx) { } std::any ParserVisitor::visitCreateMap(CelParser::CreateMapContext* ctx) { - int64_t struct_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + int64_t struct_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); std::vector entries; if (ctx->entries) { entries = visitEntries(ctx->entries); diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 33c52b1d2..1add80f84 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1520,6 +1520,29 @@ TEST_P(ExpressionTest, Parse) { } } +TEST(ExpressionTest, CompositeExpressionOffsets) { + ParserOptions options; + std::vector macros = Macro::AllMacros(); + + std::string list_expr = "[1, 2]"; + auto list_result = EnrichedParse(list_expr, macros, "", options); + ASSERT_THAT(list_result, IsOk()); + auto list_offsets = list_result->enriched_source_info().offsets(); + EXPECT_EQ(list_offsets.at(1), std::make_pair(0, 5)); + + std::string map_expr = "{'a': 1}"; + auto map_result = EnrichedParse(map_expr, macros, "", options); + ASSERT_THAT(map_result, IsOk()); + auto map_offsets = map_result->enriched_source_info().offsets(); + EXPECT_EQ(map_offsets.at(1), std::make_pair(0, 7)); + + std::string msg_expr = "Msg{f: 1}"; + auto msg_result = EnrichedParse(msg_expr, macros, "", options); + ASSERT_THAT(msg_result, IsOk()); + auto msg_offsets = msg_result->enriched_source_info().offsets(); + EXPECT_EQ(msg_offsets.at(1), std::make_pair(0, 8)); +} + TEST(ExpressionTest, TsanOom) { Parse( "[[a([[???[a[[??[a([[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" From e51522fe9eaea8d5005ee58b4385b16ee642146f Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 11 Jun 2026 10:53:19 -0700 Subject: [PATCH 061/350] Update abbreviation / import validation. More closely match the validation in the go and java implementations. PiperOrigin-RevId: 930622025 --- common/container.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/container.cc b/common/container.cc index f69f0cc80..e1db8f86c 100644 --- a/common/container.cc +++ b/common/container.cc @@ -19,6 +19,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "internal/lexis.h" @@ -92,9 +93,11 @@ absl::Status ExpressionContainer::SetContainer(absl::string_view name) { } absl::Status ExpressionContainer::AddAbbreviation(absl::string_view abrev) { + abrev = absl::StripAsciiWhitespace(abrev); if (!IsValidQualifiedName(abrev)) { return absl::InvalidArgumentError( - absl::StrCat("invalid qualified name: ", abrev)); + absl::StrCat("invalid qualified name: ", abrev, + ", wanted name of the form 'qualified.name'")); } auto pos = abrev.rfind('.'); From 6975536ba104d314fcaab2cca1cfa179239554c9 Mon Sep 17 00:00:00 2001 From: Tristan Swadell Date: Thu, 11 Jun 2026 11:02:18 -0700 Subject: [PATCH 062/350] Fix variadic logical operator planning PiperOrigin-RevId: 930627361 --- conformance/BUILD | 23 +++++-- conformance/run.bzl | 17 ++++-- conformance/run.cc | 5 ++ conformance/service.cc | 46 +++++++++----- conformance/service.h | 1 + eval/compiler/BUILD | 1 + eval/compiler/flat_expr_builder.cc | 80 +++++++++++++------------ eval/compiler/flat_expr_builder_test.cc | 7 ++- 8 files changed, 113 insertions(+), 67 deletions(-) diff --git a/conformance/BUILD b/conformance/BUILD index a6f25e001..35d554c7b 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -32,7 +32,6 @@ cc_library( "//common:ast", "//common:ast_proto", "//common:decl_proto_v1alpha1", - "//common:expr", "//common:source", "//common:value", "//common/internal:value_conversion", @@ -57,8 +56,6 @@ cc_library( "//extensions/protobuf:enum_adapter", "//internal:status_macros", "//parser", - "//parser:macro", - "//parser:macro_expr_factory", "//parser:macro_registry", "//parser:options", "//parser:standard_macros", @@ -75,8 +72,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", @@ -302,6 +297,24 @@ gen_conformance_tests( skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_CHECKED, ) +gen_conformance_tests( + name = "conformance_variadic", + checked = True, + data = _ALL_TESTS, + enable_variadic_logical_operators = True, + modern = True, + skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_CHECKED, +) + +gen_conformance_tests( + name = "conformance_legacy_variadic", + checked = True, + data = _ALL_TESTS, + enable_variadic_logical_operators = True, + modern = False, + skip_tests = _TESTS_TO_SKIP_LEGACY + _TESTS_TO_SKIP_CHECKED, +) + # Generates a bunch of `cc_test` whose names follow the pattern # `conformance_dashboard_..._{arena|refcount}_{optimized|unoptimized}_{recursive|iterative}`. gen_conformance_tests( diff --git a/conformance/run.bzl b/conformance/run.bzl index 2c0b51c0e..8faeb6c16 100644 --- a/conformance/run.bzl +++ b/conformance/run.bzl @@ -56,7 +56,7 @@ def _conformance_test_name(name, optimize, recursive): ], ) -def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard): +def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators): args = [] if modern: args.append("--modern") @@ -72,12 +72,14 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, args.append("--noskip_check") if dashboard: args.append("--dashboard") + if enable_variadic_logical_operators: + args.append("--enable_variadic_logical_operators") return args -def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard): +def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard, enable_variadic_logical_operators): cc_test( name = _conformance_test_name(name, optimize, recursive), - args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard) + ["$(rlocationpath {})".format(test) for test in data], + args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators) + ["$(rlocationpath {})".format(test) for test in data], env = select( { "@platforms//os:windows": {"CEL_SKIP_TESTS": ",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS)}, @@ -89,18 +91,20 @@ def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_ tags = tags, ) -def gen_conformance_tests(name, data, modern = False, checked = False, select_opt = False, dashboard = False, skip_tests = [], tags = []): +def gen_conformance_tests(name, data, modern = False, checked = False, select_opt = False, dashboard = False, skip_tests = [], tags = [], enable_variadic_logical_operators = False): """Generates conformance tests. Args: name: prefix for all tests + data: textproto targets describing conformance tests modern: run using modern APIs checked: whether to apply type checking - data: textproto targets describing conformance tests + select_opt: enable select optimization + dashboard: enable dashboard mode skip_tests: tests to skip in the format of the cel-spec test runner. See documentation in github.com/google/cel-spec/tests/simple/simple_test.go tags: tags added to the generated targets - dashboard: enable dashboard mode + enable_variadic_logical_operators: enable variadic logical operators """ skip_check = not checked tests = [] @@ -119,6 +123,7 @@ def gen_conformance_tests(name, data, modern = False, checked = False, select_op skip_tests = _expand_tests_to_skip(skip_tests), tags = tags, dashboard = dashboard, + enable_variadic_logical_operators = enable_variadic_logical_operators, ) native.test_suite( name = name, diff --git a/conformance/run.cc b/conformance/run.cc index 4a0493494..1be16ba60 100644 --- a/conformance/run.cc +++ b/conformance/run.cc @@ -66,6 +66,9 @@ ABSL_FLAG(std::vector, skip_tests, {}, "Tests to skip"); ABSL_FLAG(bool, dashboard, false, "Dashboard mode, ignore test failures"); ABSL_FLAG(bool, skip_check, true, "Skip type checking the expressions"); ABSL_FLAG(bool, select_optimization, false, "Enable select optimization."); +ABSL_FLAG(bool, enable_variadic_logical_operators, false, + "Enable parsing logical AND & OR operators as a single flat variadic " + "call."); namespace { @@ -261,6 +264,8 @@ NewConformanceServiceFromFlags() { .modern = absl::GetFlag(FLAGS_modern), .recursive = absl::GetFlag(FLAGS_recursive), .select_optimization = absl::GetFlag(FLAGS_select_optimization), + .enable_variadic_logical_operators = + absl::GetFlag(FLAGS_enable_variadic_logical_operators), }); ABSL_CHECK_OK(status_or_service); return std::shared_ptr( diff --git a/conformance/service.cc b/conformance/service.cc index 7e3eded82..d81200cad 100644 --- a/conformance/service.cc +++ b/conformance/service.cc @@ -128,13 +128,15 @@ cel::expr::Expr ExtractExpr( absl::Status LegacyParse(const conformance::v1alpha1::ParseRequest& request, conformance::v1alpha1::ParseResponse& response, - bool enable_optional_syntax) { + bool enable_optional_syntax, + bool enable_variadic_logical_operators) { if (request.cel_source().empty()) { return absl::InvalidArgumentError("no source code"); } cel::ParserOptions options; options.enable_optional_syntax = enable_optional_syntax; options.enable_quoted_identifiers = true; + options.enable_variadic_logical_operators = enable_variadic_logical_operators; cel::MacroRegistry macros; CEL_RETURN_IF_ERROR(cel::RegisterStandardMacros(macros, options)); CEL_RETURN_IF_ERROR( @@ -236,7 +238,8 @@ absl::Status CheckImpl(google::protobuf::Arena* arena, class LegacyConformanceServiceImpl : public ConformanceServiceInterface { public: static absl::StatusOr> Create( - bool optimize, bool recursive, bool select_optimization) { + bool optimize, bool recursive, bool select_optimization, + bool enable_variadic_logical_operators) { static auto* constant_arena = new Arena(); google::protobuf::LinkMessageReflection< @@ -313,14 +316,15 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { CEL_RETURN_IF_ERROR(cel::extensions::RegisterMathExtensionFunctions( builder->GetRegistry(), options)); - return absl::WrapUnique( - new LegacyConformanceServiceImpl(std::move(builder))); + return absl::WrapUnique(new LegacyConformanceServiceImpl( + std::move(builder), enable_variadic_logical_operators)); } void Parse(const conformance::v1alpha1::ParseRequest& request, conformance::v1alpha1::ParseResponse& response) override { auto status = - LegacyParse(request, response, /*enable_optional_syntax=*/false); + LegacyParse(request, response, /*enable_optional_syntax=*/false, + enable_variadic_logical_operators_); if (!status.ok()) { auto* issue = response.add_issues(); issue->set_code(ToGrpcCode(status.code())); @@ -418,17 +422,20 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { } private: - explicit LegacyConformanceServiceImpl( - std::unique_ptr builder) - : builder_(std::move(builder)) {} + LegacyConformanceServiceImpl(std::unique_ptr builder, + bool enable_variadic_logical_operators) + : builder_(std::move(builder)), + enable_variadic_logical_operators_(enable_variadic_logical_operators) {} std::unique_ptr builder_; + bool enable_variadic_logical_operators_; }; class ModernConformanceServiceImpl : public ConformanceServiceInterface { public: static absl::StatusOr> Create( - bool optimize, bool recursive, bool select_optimization) { + bool optimize, bool recursive, bool select_optimization, + bool enable_variadic_logical_operators) { google::protobuf::LinkMessageReflection< cel::expr::conformance::proto3::TestAllTypes>(); google::protobuf::LinkMessageReflection< @@ -470,8 +477,9 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { options.max_recursion_depth = 48; } - return absl::WrapUnique(new ModernConformanceServiceImpl( - options, optimize, select_optimization)); + return absl::WrapUnique( + new ModernConformanceServiceImpl(options, optimize, select_optimization, + enable_variadic_logical_operators)); } absl::StatusOr> Setup( @@ -523,7 +531,8 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { void Parse(const conformance::v1alpha1::ParseRequest& request, conformance::v1alpha1::ParseResponse& response) override { auto status = - LegacyParse(request, response, /*enable_optional_syntax=*/true); + LegacyParse(request, response, /*enable_optional_syntax=*/true, + enable_variadic_logical_operators_); if (!status.ok()) { auto* issue = response.add_issues(); issue->set_code(ToGrpcCode(status.code())); @@ -614,10 +623,12 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { private: ModernConformanceServiceImpl(const RuntimeOptions& options, bool enable_optimizations, - bool enable_select_optimization) + bool enable_select_optimization, + bool enable_variadic_logical_operators) : options_(options), enable_optimizations_(enable_optimizations), - enable_select_optimization_(enable_select_optimization) {} + enable_select_optimization_(enable_select_optimization), + enable_variadic_logical_operators_(enable_variadic_logical_operators) {} static absl::StatusOr> Plan( const cel::Runtime& runtime, @@ -648,6 +659,7 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { RuntimeOptions options_; bool enable_optimizations_; bool enable_select_optimization_; + bool enable_variadic_logical_operators_; }; } // namespace @@ -660,10 +672,12 @@ absl::StatusOr> NewConformanceService(const ConformanceServiceOptions& options) { if (options.modern) { return google::api::expr::runtime::ModernConformanceServiceImpl::Create( - options.optimize, options.recursive, options.select_optimization); + options.optimize, options.recursive, options.select_optimization, + options.enable_variadic_logical_operators); } else { return google::api::expr::runtime::LegacyConformanceServiceImpl::Create( - options.optimize, options.recursive, options.select_optimization); + options.optimize, options.recursive, options.select_optimization, + options.enable_variadic_logical_operators); } } diff --git a/conformance/service.h b/conformance/service.h index 2dd2abf32..8eb97296e 100644 --- a/conformance/service.h +++ b/conformance/service.h @@ -46,6 +46,7 @@ struct ConformanceServiceOptions { bool arena; bool recursive; bool select_optimization; + bool enable_variadic_logical_operators = false; }; absl::StatusOr> diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index ed8e4d20c..f7300cb58 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -193,6 +193,7 @@ cc_test( "//internal:status_macros", "//internal:testing", "//parser", + "//parser:options", "//runtime:function", "//runtime:function_adapter", "//runtime:runtime_options", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index d6ccdf040..fc6d87b16 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -2154,7 +2154,7 @@ void BinaryCondVisitor::PreVisit(const cel::Expr* expr) { case BinaryCond::kOr: visitor_->ValidateOrError( !expr->call_expr().has_target() && - expr->call_expr().args().size() == 2, + expr->call_expr().args().size() >= 2, "Invalid argument count for a binary function call."); break; case BinaryCond::kOptionalOr: @@ -2172,28 +2172,40 @@ void BinaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { return; } const int last_arg_index = expr->call_expr().args().size() - 1; - if (short_circuiting_ && arg_num < last_arg_index && - (cond_ == BinaryCond::kAnd || cond_ == BinaryCond::kOr)) { - // If first branch evaluation result is enough to determine output, - // jump over the second branch and provide result of the first argument as - // final output. - // Retain pointers to the jump steps so we can update the target after - // planning the next arguments. - std::unique_ptr jump_step; - switch (cond_) { - case BinaryCond::kAnd: - jump_step = CreateCondJumpStep(false, true, {}, expr->id()); - break; - case BinaryCond::kOr: - jump_step = CreateCondJumpStep(true, true, {}, expr->id()); - break; - default: - ABSL_UNREACHABLE(); + if (cond_ == BinaryCond::kAnd || cond_ == BinaryCond::kOr) { + if (arg_num > 0) { + switch (cond_) { + case BinaryCond::kAnd: + visitor_->AddStep(CreateAndStep(expr->id())); + break; + case BinaryCond::kOr: + visitor_->AddStep(CreateOrStep(expr->id())); + break; + default: + break; + } + if (short_circuiting_ && !jump_steps_.empty()) { + visitor_->SetProgressStatusIfError( + jump_steps_.back().set_target(visitor_->GetCurrentIndex())); + } } - ProgramStepIndex index = visitor_->GetCurrentIndex(); - if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); - jump_step_ptr) { - jump_steps_.push_back(Jump(index, jump_step_ptr)); + if (short_circuiting_ && arg_num < last_arg_index) { + std::unique_ptr jump_step; + switch (cond_) { + case BinaryCond::kAnd: + jump_step = CreateCondJumpStep(false, true, {}, expr->id()); + break; + case BinaryCond::kOr: + jump_step = CreateCondJumpStep(true, true, {}, expr->id()); + break; + default: + ABSL_UNREACHABLE(); + } + ProgramStepIndex index = visitor_->GetCurrentIndex(); + if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); + jump_step_ptr) { + jump_steps_.push_back(Jump(index, jump_step_ptr)); + } } } } @@ -2251,17 +2263,9 @@ void BinaryCondVisitor::PostVisit(const cel::Expr* expr) { return; } - int args_count = (cond_ == BinaryCond::kAnd || cond_ == BinaryCond::kOr) - ? expr->call_expr().args().size() - : 2; - for (int i = 0; i < args_count - 1; ++i) { + if (cond_ == BinaryCond::kOptionalOr || + cond_ == BinaryCond::kOptionalOrValue) { switch (cond_) { - case BinaryCond::kAnd: - visitor_->AddStep(CreateAndStep(expr->id())); - break; - case BinaryCond::kOr: - visitor_->AddStep(CreateOrStep(expr->id())); - break; case BinaryCond::kOptionalOr: visitor_->AddStep( CreateOptionalOrStep(/*is_or_value=*/false, expr->id())); @@ -2273,13 +2277,11 @@ void BinaryCondVisitor::PostVisit(const cel::Expr* expr) { default: ABSL_UNREACHABLE(); } - } - if (short_circuiting_) { - // If short-circuiting is enabled, point the conditional jump past the - // boolean operator step. - for (auto& jump : jump_steps_) { - visitor_->SetProgressStatusIfError( - jump.set_target(visitor_->GetCurrentIndex())); + if (short_circuiting_) { + for (auto& jump : jump_steps_) { + visitor_->SetProgressStatusIfError( + jump.set_target(visitor_->GetCurrentIndex())); + } } } } diff --git a/eval/compiler/flat_expr_builder_test.cc b/eval/compiler/flat_expr_builder_test.cc index e2581e3fd..105060282 100644 --- a/eval/compiler/flat_expr_builder_test.cc +++ b/eval/compiler/flat_expr_builder_test.cc @@ -64,6 +64,7 @@ #include "internal/proto_matchers.h" #include "internal/status_macros.h" #include "internal/testing.h" +#include "parser/options.h" #include "parser/parser.h" #include "runtime/function.h" #include "runtime/function_adapter.h" @@ -2916,7 +2917,11 @@ class FlatExprBuilderVariadicLogicalTest TEST_P(FlatExprBuilderVariadicLogicalTest, Evaluate) { const auto& test_case = GetParam(); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(test_case.expr)); + parser::ParserOptions parser_options; + parser_options.enable_variadic_logical_operators = true; + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + parser::Parse(test_case.expr, test_case.label, parser_options)); cel::RuntimeOptions options; options.unknown_processing = From c0e43073ea7f4fbbcdc59b45f34abe8bd5b9364a Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 11 Jun 2026 11:22:17 -0700 Subject: [PATCH 063/350] refactor: Move type formatter to common. PiperOrigin-RevId: 930639404 --- checker/internal/BUILD | 15 ++---------- checker/internal/type_checker_impl.cc | 2 +- checker/internal/type_inference_context.cc | 18 +++++++-------- common/BUILD | 23 +++++++++++++++++++ .../internal => common}/format_type_name.cc | 6 ++--- .../internal => common}/format_type_name.h | 10 ++++---- .../format_type_name_test.cc | 6 ++--- 7 files changed, 46 insertions(+), 34 deletions(-) rename {checker/internal => common}/format_type_name.cc (97%) rename {checker/internal => common}/format_type_name.h (74%) rename {checker/internal => common}/format_type_name_test.cc (97%) diff --git a/checker/internal/BUILD b/checker/internal/BUILD index 777457830..20c476db2 100644 --- a/checker/internal/BUILD +++ b/checker/internal/BUILD @@ -130,7 +130,6 @@ cc_library( "type_checker_impl.h", ], deps = [ - ":format_type_name", ":namespace_generator", ":proto_type_mask", ":type_check_env", @@ -149,6 +148,7 @@ cc_library( "//common:container", "//common:decl", "//common:expr", + "//common:format_type_name", "//common:standard_definitions", "//common:type", "//common:type_kind", @@ -243,8 +243,8 @@ cc_library( srcs = ["type_inference_context.cc"], hdrs = ["type_inference_context.h"], deps = [ - ":format_type_name", "//common:decl", + "//common:format_type_name", "//common:standard_definitions", "//common:type", "//common:type_kind", @@ -275,17 +275,6 @@ cc_test( ], ) -cc_library( - name = "format_type_name", - srcs = ["format_type_name.cc"], - hdrs = ["format_type_name.h"], - deps = [ - "//common:type", - "//common:type_kind", - "@com_google_absl//absl/strings", - ], -) - cc_library( name = "descriptor_pool_type_introspector", srcs = ["descriptor_pool_type_introspector.cc"], diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 6b6b051b1..f3a06a28d 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -36,7 +36,6 @@ #include "absl/types/optional.h" #include "absl/types/span.h" #include "checker/checker_options.h" -#include "checker/internal/format_type_name.h" #include "checker/internal/namespace_generator.h" #include "checker/internal/type_check_env.h" #include "checker/internal/type_checker_builder_impl.h" @@ -52,6 +51,7 @@ #include "common/constant.h" #include "common/decl.h" #include "common/expr.h" +#include "common/format_type_name.h" #include "common/standard_definitions.h" #include "common/type.h" #include "common/type_kind.h" diff --git a/checker/internal/type_inference_context.cc b/checker/internal/type_inference_context.cc index 1a87d9e15..4681784af 100644 --- a/checker/internal/type_inference_context.cc +++ b/checker/internal/type_inference_context.cc @@ -28,8 +28,8 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" -#include "checker/internal/format_type_name.h" #include "common/decl.h" +#include "common/format_type_name.h" #include "common/standard_definitions.h" #include "common/type.h" #include "common/type_kind.h" @@ -657,14 +657,14 @@ bool TypeInferenceContext::AssignabilityContext::IsAssignable(const Type& from, std::string TypeInferenceContext::DebugString() const { return absl::StrCat( "type_parameter_bindings: ", - absl::StrJoin( - type_parameter_bindings_, "\n ", - [](std::string* out, const auto& binding) { - absl::StrAppend( - out, binding.first, " (", binding.second.name, ") -> ", - checker_internal::FormatTypeName( - binding.second.type.value_or(Type(TypeParamType("none"))))); - })); + absl::StrJoin(type_parameter_bindings_, "\n ", + [](std::string* out, const auto& binding) { + absl::StrAppend( + out, binding.first, " (", binding.second.name, + ") -> ", + cel::FormatTypeName(binding.second.type.value_or( + Type(TypeParamType("none"))))); + })); } void TypeInferenceContext::AssignabilityContext:: diff --git a/common/BUILD b/common/BUILD index f7c897e57..93410306f 100644 --- a/common/BUILD +++ b/common/BUILD @@ -601,6 +601,17 @@ cc_library( ], ) +cc_library( + name = "format_type_name", + srcs = ["format_type_name.cc"], + hdrs = ["format_type_name.h"], + deps = [ + ":type", + ":type_kind", + "@com_google_absl//absl/strings", + ], +) + cc_test( name = "type_test", srcs = glob([ @@ -623,6 +634,18 @@ cc_test( ], ) +cc_test( + name = "format_type_name_test", + srcs = ["format_type_name_test.cc"], + deps = [ + ":format_type_name", + ":type", + "//internal:testing", + "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "value", srcs = glob( diff --git a/checker/internal/format_type_name.cc b/common/format_type_name.cc similarity index 97% rename from checker/internal/format_type_name.cc rename to common/format_type_name.cc index 7cd17251f..4bd6c2e61 100644 --- a/checker/internal/format_type_name.cc +++ b/common/format_type_name.cc @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "checker/internal/format_type_name.h" +#include "common/format_type_name.h" #include #include @@ -20,7 +20,7 @@ #include "common/type.h" #include "common/type_kind.h" -namespace cel::checker_internal { +namespace cel { namespace { struct FormatImplRecord { @@ -177,4 +177,4 @@ std::string FormatTypeName(const Type& type) { return out; } -} // namespace cel::checker_internal +} // namespace cel diff --git a/checker/internal/format_type_name.h b/common/format_type_name.h similarity index 74% rename from checker/internal/format_type_name.h rename to common/format_type_name.h index c31e1c4d0..723ac20fd 100644 --- a/checker/internal/format_type_name.h +++ b/common/format_type_name.h @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_FORMAT_TYPE_NAME_H_ -#define THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_FORMAT_TYPE_NAME_H_ +#ifndef THIRD_PARTY_CEL_CPP_COMMON_FORMAT_TYPE_NAME_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_FORMAT_TYPE_NAME_H_ #include #include "common/type.h" -namespace cel::checker_internal { +namespace cel { // Format the type name for presentation in error messages. Matches the // formatting used in github.com/cel-spec. std::string FormatTypeName(const Type& type); -} // namespace cel::checker_internal +} // namespace cel -#endif // THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_FORMAT_TYPE_NAME_H_ +#endif // THIRD_PARTY_CEL_CPP_COMMON_FORMAT_TYPE_NAME_H_ diff --git a/checker/internal/format_type_name_test.cc b/common/format_type_name_test.cc similarity index 97% rename from checker/internal/format_type_name_test.cc rename to common/format_type_name_test.cc index ff04e04d2..ca63f60b0 100644 --- a/checker/internal/format_type_name_test.cc +++ b/common/format_type_name_test.cc @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "checker/internal/format_type_name.h" +#include "common/format_type_name.h" #include "common/type.h" #include "internal/testing.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "google/protobuf/arena.h" -namespace cel::checker_internal { +namespace cel { namespace { using ::cel::expr::conformance::proto2::GlobalEnum_descriptor; @@ -115,4 +115,4 @@ TEST(FormatTypeNameTest, ArbitraryNesting) { #endif } // namespace -} // namespace cel::checker_internal +} // namespace cel From d2a8a7831b76fb128de4d03254bc8f81dfcf5652 Mon Sep 17 00:00:00 2001 From: Tristan Swadell Date: Thu, 11 Jun 2026 12:12:47 -0700 Subject: [PATCH 064/350] Create a `proto_to_predicate` compiler for converting proto messages into CEL expressions PiperOrigin-RevId: 930668150 --- common/expr_factory.h | 5 + internal/json.h | 4 +- tools/BUILD | 50 +++ tools/proto_to_predicate.cc | 459 ++++++++++++++++++++++++ tools/proto_to_predicate.h | 48 +++ tools/proto_to_predicate_test.cc | 593 +++++++++++++++++++++++++++++++ tools/testdata/BUILD | 19 +- tools/testdata/test_policy.proto | 73 ++++ 8 files changed, 1245 insertions(+), 6 deletions(-) create mode 100644 tools/proto_to_predicate.cc create mode 100644 tools/proto_to_predicate.h create mode 100644 tools/proto_to_predicate_test.cc create mode 100644 tools/testdata/test_policy.proto diff --git a/common/expr_factory.h b/common/expr_factory.h index 5607d8deb..757318545 100644 --- a/common/expr_factory.h +++ b/common/expr_factory.h @@ -34,6 +34,10 @@ class MacroExprFactory; class ParserMacroExprFactory; class OptimizerExprFactory; +namespace tools { +class ProtoToPredicateBuilder; +} + class ExprFactory { protected: // `IsExprLike` determines whether `T` is some `Expr`. Currently that means @@ -380,6 +384,7 @@ class ExprFactory { friend class MacroExprFactory; friend class ParserMacroExprFactory; friend class OptimizerExprFactory; + friend class tools::ProtoToPredicateBuilder; ExprFactory() : accu_var_(kAccumulatorVariableName) {} diff --git a/internal/json.h b/internal/json.h index d32c42741..e35909d0e 100644 --- a/internal/json.h +++ b/internal/json.h @@ -26,7 +26,7 @@ namespace cel::internal { // Converts the given message to its `google.protobuf.Value` equivalent -// representation. This is similar to `proto2::json::MessageToJsonString()`, +// representation. This is similar to `google::protobuf::json::MessageToJsonString()`, // except that this results in structured serialization. absl::Status MessageToJson( const google::protobuf::Message& message, @@ -45,7 +45,7 @@ absl::Status MessageToJson( google::protobuf::Message* absl_nonnull result); // Converts the given message field to its `google.protobuf.Value` equivalent -// representation. This is similar to `proto2::json::MessageToJsonString()`, +// representation. This is similar to `google::protobuf::json::MessageToJsonString()`, // except that this results in structured serialization. absl::Status MessageFieldToJson( const google::protobuf::Message& message, diff --git a/tools/BUILD b/tools/BUILD index ceb2befc5..af006a67b 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -204,6 +204,56 @@ cc_library( ], ) +cc_library( + name = "proto_to_predicate", + srcs = ["proto_to_predicate.cc"], + hdrs = ["proto_to_predicate.h"], + deps = [ + "//common:ast", + "//common:expr", + "//common:expr_factory", + "//common:operators", + "//internal:status_macros", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "proto_to_predicate_test", + srcs = ["proto_to_predicate_test.cc"], + deps = [ + ":cel_unparser", + ":proto_to_predicate", + "//common:ast", + "//common:ast_proto", + "//common:value", + "//env:config", + "//env:env_runtime", + "//env:env_yaml", + "//env:runtime_std_extensions", + "//eval/testutil:test_message_cc_proto", + "//extensions/protobuf:value", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//parser", + "//runtime", + "//runtime:activation", + "//tools/testdata:test_policy_cc_proto", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + cc_test( name = "descriptor_pool_builder_test", srcs = ["descriptor_pool_builder_test.cc"], diff --git a/tools/proto_to_predicate.cc b/tools/proto_to_predicate.cc new file mode 100644 index 000000000..8c89ee2f0 --- /dev/null +++ b/tools/proto_to_predicate.cc @@ -0,0 +1,459 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tools/proto_to_predicate.h" + +#include +#include +#include +#include +#include +#include + +#include "google/protobuf/descriptor.pb.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/operators.h" +#include "internal/status_macros.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" +#include "google/protobuf/reflection.h" + +namespace cel::tools { + +using ::google::api::expr::common::CelOperator; +using ::google::protobuf::FieldDescriptor; +using ::google::protobuf::Message; +using ::google::protobuf::Reflection; + +class ProtoToPredicateBuilder final : private ExprFactory { + public: + ProtoToPredicateBuilder() : id_(1) {} + + absl::StatusOr Build(absl::string_view input_name, + const Message& message) { + std::vector predicates; + Expr base_expr = NewIdent(NextId(), input_name); + + CEL_RETURN_IF_ERROR(Walk(message, base_expr, predicates)); + Expr root = LogicalAnd(predicates); + return Ast(std::move(root), std::move(source_info_)); + } + + absl::StatusOr Build(absl::string_view input_name, + absl::Span messages) { + if (messages.empty()) { + return Ast(NewBoolConst(NextId(), true), std::move(source_info_)); + } + + std::vector message_asts; + message_asts.reserve(messages.size()); + for (const auto* message : messages) { + std::vector predicates; + Expr base_expr = NewIdent(NextId(), input_name); + + CEL_RETURN_IF_ERROR(Walk(*message, base_expr, predicates)); + message_asts.push_back(LogicalAnd(predicates)); + } + + return Ast(LogicalOr(message_asts), std::move(source_info_)); + } + + private: + // Retrieves the "match_path" string option from the field options if + // defined, returning an empty string otherwise. + std::string GetMatchPath(const ::google::protobuf::FieldDescriptor* field) { + const ::google::protobuf::Message& options = field->options(); + const ::google::protobuf::Reflection* refl = options.GetReflection(); + std::vector fields; + refl->ListFields(options, &fields); + for (const auto* f : fields) { + if (f->name() == "match_path") { + return refl->GetString(options, f); + } + } + return ""; + } + + // Parses a dot-separated string representation of a path (e.g. "dest.region") + // and builds a corresponding select chain AST. + Expr ParseAndBuildPath(absl::string_view path_str) { + std::vector parts = absl::StrSplit(path_str, '.'); + Expr e = NewIdent(NextId(), parts[0]); + for (size_t i = 1; i < parts.size(); ++i) { + e = NewSelect(NextId(), std::move(e), parts[i]); + } + return e; + } + ExprId NextId() { return id_++; } + + // --------------------------------------------------------------------------- + // Field value extraction + // --------------------------------------------------------------------------- + + // Converts a singular field value to a CEL constant expression. + Expr PrimitiveToExpr(ExprId expr_id, const Message& message, + const Reflection* reflection, + const FieldDescriptor* field) { + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return NewIntConst(expr_id, reflection->GetInt32(message, field)); + case FieldDescriptor::CPPTYPE_INT64: + return NewIntConst(expr_id, reflection->GetInt64(message, field)); + case FieldDescriptor::CPPTYPE_UINT32: + return NewUintConst(expr_id, reflection->GetUInt32(message, field)); + case FieldDescriptor::CPPTYPE_UINT64: + return NewUintConst(expr_id, reflection->GetUInt64(message, field)); + case FieldDescriptor::CPPTYPE_DOUBLE: + return NewDoubleConst(expr_id, reflection->GetDouble(message, field)); + case FieldDescriptor::CPPTYPE_FLOAT: + return NewDoubleConst(expr_id, reflection->GetFloat(message, field)); + case FieldDescriptor::CPPTYPE_BOOL: + return NewBoolConst(expr_id, reflection->GetBool(message, field)); + case FieldDescriptor::CPPTYPE_ENUM: + return NewIntConst(expr_id, reflection->GetEnumValue(message, field)); + case FieldDescriptor::CPPTYPE_STRING: { + std::string str_val = reflection->GetString(message, field); + if (field->type() == FieldDescriptor::TYPE_BYTES) { + return NewBytesConst(expr_id, std::move(str_val)); + } + return NewStringConst(expr_id, std::move(str_val)); + } + default: + // Log a warning as message should be handled by Walk. + ABSL_LOG(WARNING) << "PrimitiveToExpr: Unhandled field type: " + << FieldDescriptor::TypeName(field->type()); + break; + } + return NewNullConst(expr_id); + } + + Expr PrimitiveToExpr(const Message& message, const Reflection* reflection, + const FieldDescriptor* field) { + return PrimitiveToExpr(NextId(), message, reflection, field); + } + + // Converts a repeated field element to a CEL constant expression. + Expr RepeatedPrimitiveToExpr(const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, int index) { + const ExprId id = NextId(); + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return NewIntConst(id, + reflection->GetRepeatedInt32(message, field, index)); + case FieldDescriptor::CPPTYPE_INT64: + return NewIntConst(id, + reflection->GetRepeatedInt64(message, field, index)); + case FieldDescriptor::CPPTYPE_UINT32: + return NewUintConst( + id, reflection->GetRepeatedUInt32(message, field, index)); + case FieldDescriptor::CPPTYPE_UINT64: + return NewUintConst( + id, reflection->GetRepeatedUInt64(message, field, index)); + case FieldDescriptor::CPPTYPE_DOUBLE: + return NewDoubleConst( + id, reflection->GetRepeatedDouble(message, field, index)); + case FieldDescriptor::CPPTYPE_FLOAT: + return NewDoubleConst( + id, reflection->GetRepeatedFloat(message, field, index)); + case FieldDescriptor::CPPTYPE_BOOL: + return NewBoolConst(id, + reflection->GetRepeatedBool(message, field, index)); + case FieldDescriptor::CPPTYPE_ENUM: + return NewIntConst( + id, reflection->GetRepeatedEnumValue(message, field, index)); + case FieldDescriptor::CPPTYPE_STRING: { + std::string str_val = + reflection->GetRepeatedString(message, field, index); + if (field->type() == FieldDescriptor::TYPE_BYTES) { + return NewBytesConst(id, std::move(str_val)); + } + return NewStringConst(id, std::move(str_val)); + } + default: + break; + } + return NewNullConst(id); + } + + // --------------------------------------------------------------------------- + // Expression construction helpers + // --------------------------------------------------------------------------- + + // Creates a binary operator call: `lhs rhs`. + Expr ConstructBinaryOp(absl::string_view op, Expr lhs, Expr rhs) { + std::vector args = {std::move(lhs), std::move(rhs)}; + return NewCall(NextId(), op, std::move(args)); + } + + Expr ConstructEquality(Expr lhs, Expr rhs) { + return ConstructBinaryOp(CelOperator::EQUALS, std::move(lhs), + std::move(rhs)); + } + + Expr LogicalOr(std::vector& exprs) { + return LogicalOp(CelOperator::LOGICAL_OR, exprs); + } + + Expr LogicalAnd(std::vector& exprs) { + return LogicalOp(CelOperator::LOGICAL_AND, exprs); + } + + // Left-folds a vector of expressions with a binary operator. + // Requires: `exprs` is non-empty. + Expr LogicalOp(absl::string_view op, std::vector& exprs) { + if (exprs.empty()) { + return NewBoolConst(NextId(), true); + } + if (exprs.size() == 1) { + return std::move(exprs[0]); + } + return NewCall(NextId(), op, std::move(exprs)); + } + + // --------------------------------------------------------------------------- + // Map field predicate (extracted from Walk for readability) + // --------------------------------------------------------------------------- + + // Builds the predicate for a map field to assert that all key-value pairs + // specified in the policy are present in the input map field: + // "key" in input.map && input.map["key"] == value + absl::Status WalkMapField(const Reflection* reflection, + const Message& message, + const FieldDescriptor* field, const Expr& base_expr, + int size, std::vector& predicates) { + const FieldDescriptor* const key_field = + field->message_type()->FindFieldByName("key"); + const FieldDescriptor* const value_field = + field->message_type()->FindFieldByName("value"); + + Expr map_path = NewSelect(NextId(), base_expr, field->name()); + + struct MapEntry { + const Message* message; + }; + std::vector entries; + entries.reserve(size); + for (int i = 0; i < size; ++i) { + entries.push_back({&reflection->GetRepeatedMessage(message, field, i)}); + } + + if (!entries.empty()) { + const Reflection* const entry_ref = entries[0].message->GetReflection(); + std::sort(entries.begin(), entries.end(), + [entry_ref, key_field](const MapEntry& a, const MapEntry& b) { + switch (key_field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return entry_ref->GetInt32(*a.message, key_field) < + entry_ref->GetInt32(*b.message, key_field); + case FieldDescriptor::CPPTYPE_INT64: + return entry_ref->GetInt64(*a.message, key_field) < + entry_ref->GetInt64(*b.message, key_field); + case FieldDescriptor::CPPTYPE_UINT32: + return entry_ref->GetUInt32(*a.message, key_field) < + entry_ref->GetUInt32(*b.message, key_field); + case FieldDescriptor::CPPTYPE_UINT64: + return entry_ref->GetUInt64(*a.message, key_field) < + entry_ref->GetUInt64(*b.message, key_field); + case FieldDescriptor::CPPTYPE_BOOL: + return !entry_ref->GetBool(*a.message, key_field) && + entry_ref->GetBool(*b.message, key_field); + case FieldDescriptor::CPPTYPE_STRING: + return entry_ref->GetString(*a.message, key_field) < + entry_ref->GetString(*b.message, key_field); + default: + return false; + } + }); + } + + std::vector map_checks; + map_checks.reserve(size); + for (const auto& entry : entries) { + const Message& entry_msg = *entry.message; + const Reflection* const entry_ref = entry_msg.GetReflection(); + + Expr key_expr = PrimitiveToExpr(entry_msg, entry_ref, key_field); + + // Represents `"key" in input.map` to assert the key exists. + Expr in_check = NewCall(NextId(), CelOperator::IN, + std::vector{key_expr, map_path}); + // Represents `input.map["key"]` to lookup the value. + Expr lookup_path = NewCall(NextId(), CelOperator::INDEX, + std::vector{map_path, key_expr}); + + if (value_field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + const Message& value_msg = + entry_ref->GetMessage(entry_msg, value_field); + std::vector val_predicates; + CEL_RETURN_IF_ERROR(Walk(value_msg, lookup_path, val_predicates)); + + if (!val_predicates.empty()) { + // Represents `"key" in input.map && (nested message fields check...)` + map_checks.push_back(std::move(in_check)); + map_checks.insert(map_checks.end(), + std::make_move_iterator(val_predicates.begin()), + std::make_move_iterator(val_predicates.end())); + } else { + // Represents `"key" in input.map` if nested message is empty. + map_checks.push_back(std::move(in_check)); + } + } else { + Expr value_expr = PrimitiveToExpr(entry_msg, entry_ref, value_field); + // Represents `input.map["key"] == value` + Expr eq_check = + ConstructEquality(std::move(lookup_path), std::move(value_expr)); + + // Represents `"key" in input.map && input.map["key"] == value` + map_checks.push_back(std::move(in_check)); + map_checks.push_back(std::move(eq_check)); + } + } + + predicates.push_back(LogicalAnd(map_checks)); + return absl::OkStatus(); + } + + // --------------------------------------------------------------------------- + // Repeated field predicate (extracted from Walk for readability) + // --------------------------------------------------------------------------- + + // Builds predicates for a repeated field: + // - Repeated Messages are mapped to a logical OR (||) of the generated + // predicates for each message. + // - Repeated Primitives are mapped either to: + // - `lhs in [values]` if a "match_path" option is specified. + // - `value in input.field` conjoined with && for each value otherwise. + absl::Status WalkRepeatedField(const Reflection* reflection, + const Message& message, + const FieldDescriptor* field, + const Expr& base_expr, int size, + std::vector& predicates) { + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + std::vector message_asts; + message_asts.reserve(size); + for (int i = 0; i < size; ++i) { + const Message& sub_message = + reflection->GetRepeatedMessage(message, field, i); + std::vector sub_predicates; + Expr sub_base = NewSelect(NextId(), base_expr, field->name()); + CEL_RETURN_IF_ERROR(Walk(sub_message, sub_base, sub_predicates)); + message_asts.push_back(LogicalAnd(sub_predicates)); + } + // Represents alternate message predicates conjoined with OR: `msg_1 || + // msg_2 || ...` + predicates.push_back(LogicalOr(message_asts)); + return absl::OkStatus(); + } + + std::vector elements; + elements.reserve(size); + for (int i = 0; i < size; ++i) { + elements.push_back(NewListElement( + RepeatedPrimitiveToExpr(message, reflection, field, i))); + } + Expr literal_list = NewList(NextId(), std::move(elements)); + + std::string match_path_val = GetMatchPath(field); + if (!match_path_val.empty()) { + Expr lhs = ParseAndBuildPath(match_path_val); + // Represents `lhs in [values]` check (e.g. `dest.region in ["us-east", + // "us-west"]`). + predicates.push_back( + NewCall(NextId(), CelOperator::IN, + std::vector{std::move(lhs), std::move(literal_list)})); + return absl::OkStatus(); + } + + Expr map_path = NewSelect(NextId(), base_expr, field->name()); + std::vector element_checks; + element_checks.reserve(size); + for (int i = 0; i < size; ++i) { + Expr elem_expr = RepeatedPrimitiveToExpr(message, reflection, field, i); + // Represents `value in input.field` check. + Expr in_check = + NewCall(NextId(), CelOperator::IN, + std::vector{std::move(elem_expr), map_path}); + element_checks.push_back(std::move(in_check)); + } + // Represents `"val1" in input.list && "val2" in input.list && ...` + predicates.push_back(LogicalAnd(element_checks)); + + return absl::OkStatus(); + } + + // --------------------------------------------------------------------------- + // Recursive message walk + // --------------------------------------------------------------------------- + + absl::Status Walk(const Message& message, const Expr& base_expr, + std::vector& predicates) { + const Reflection* const reflection = message.GetReflection(); + std::vector fields; + reflection->ListFields(message, &fields); + + for (const auto* field : fields) { + if (field->is_map()) { + const int size = reflection->FieldSize(message, field); + if (size > 0) { + CEL_RETURN_IF_ERROR(WalkMapField(reflection, message, field, + base_expr, size, predicates)); + } + } else if (field->is_repeated()) { + const int size = reflection->FieldSize(message, field); + if (size > 0) { + CEL_RETURN_IF_ERROR(WalkRepeatedField(reflection, message, field, + base_expr, size, predicates)); + } + } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + const Message& sub_message = reflection->GetMessage(message, field); + Expr field_path = NewSelect(NextId(), base_expr, field->name()); + CEL_RETURN_IF_ERROR(Walk(sub_message, field_path, predicates)); + } else { + // Primitive field: base_expr.field == + Expr field_path = NewSelect(NextId(), base_expr, field->name()); + predicates.push_back( + ConstructEquality(std::move(field_path), + PrimitiveToExpr(message, reflection, field))); + } + } + return absl::OkStatus(); + } + + ExprId id_; + SourceInfo source_info_; +}; + +absl::StatusOr ProtoToPredicateAst(absl::string_view input_name, + const ::google::protobuf::Message& message) { + ProtoToPredicateBuilder builder; + return builder.Build(input_name, message); +} + +absl::StatusOr ProtoToPredicateAst( + absl::string_view input_name, + absl::Span messages) { + ProtoToPredicateBuilder builder; + return builder.Build(input_name, messages); +} + +} // namespace cel::tools diff --git a/tools/proto_to_predicate.h b/tools/proto_to_predicate.h new file mode 100644 index 000000000..ed01cb1e8 --- /dev/null +++ b/tools/proto_to_predicate.h @@ -0,0 +1,48 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TOOLS_PROTO_TO_PREDICATE_H_ +#define THIRD_PARTY_CEL_CPP_TOOLS_PROTO_TO_PREDICATE_H_ + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "google/protobuf/message.h" + +namespace cel::tools { + +// Translates a Protocol Buffer message into a CEL AST representing a predicate. +// +// NOTE: The protocol message schemas used for policy definition should use +// `proto2` or `editions` (and not `proto3` implicit presence) to ensure correct +// behavior, as this library relies on field presence (via reflection) to +// identify which fields are explicitly set by the policy. +absl::StatusOr ProtoToPredicateAst(absl::string_view input_name, + const ::google::protobuf::Message& message); + +// Translates a list of Protocol Buffer messages into a CEL AST representing a +// conjoined or alternate predicate. +// +// NOTE: The protocol message schemas used for policy definition should use +// `proto2` or `editions` (and not `proto3` implicit presence) to ensure correct +// behavior, as this library relies on field presence (via reflection) to +// identify which fields are explicitly set by the policy. +absl::StatusOr ProtoToPredicateAst( + absl::string_view input_name, + absl::Span messages); + +} // namespace cel::tools + +#endif // THIRD_PARTY_CEL_CPP_TOOLS_PROTO_TO_PREDICATE_H_ diff --git a/tools/proto_to_predicate_test.cc b/tools/proto_to_predicate_test.cc new file mode 100644 index 000000000..80ad140c7 --- /dev/null +++ b/tools/proto_to_predicate_test.cc @@ -0,0 +1,593 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tools/proto_to_predicate.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/ast_proto.h" +#include "common/value.h" +#include "env/config.h" +#include "env/env_runtime.h" +#include "env/env_yaml.h" +#include "env/runtime_std_extensions.h" +#include "eval/testutil/test_message.pb.h" +#include "extensions/protobuf/value.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "parser/parser.h" +#include "runtime/activation.h" +#include "runtime/runtime.h" +#include "tools/cel_unparser.h" +#include "tools/testdata/test_policy.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/json/json.h" +#include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" + +namespace cel::tools { +namespace { + +using ::absl_testing::IsOk; +using ::google::api::expr::runtime::TestMessage; + +constexpr absl::string_view kEnvYaml = R"( +name: "test" +extensions: + - name: "bindings" + - name: "optional" +variables: + - name: "input" + type: "google.api.expr.runtime.TestMessage" +)"; + +TestMessage ParseTestMessage(absl::string_view textproto) { + TestMessage msg; + google::protobuf::TextFormat::ParseFromString(textproto, &msg); + return msg; +} + +absl::StatusOr EvaluatePredicate(const cel::Ast& ast, + const TestMessage& input) { + auto descriptor_pool = cel::internal::GetSharedTestingDescriptorPool(); + + CEL_ASSIGN_OR_RETURN(cel::Config config, + cel::EnvConfigFromYaml(std::string(kEnvYaml))); + + cel::EnvRuntime env_runtime; + env_runtime.SetDescriptorPool(descriptor_pool); + cel::RegisterStandardExtensions(env_runtime); + env_runtime.SetConfig(config); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + env_runtime.NewRuntime()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::make_unique(ast))); + + google::protobuf::Arena arena; + cel::Activation activation; + CEL_ASSIGN_OR_RETURN( + cel::Value val, cel::extensions::ProtoMessageToValue( + input, descriptor_pool.get(), + google::protobuf::MessageFactory::generated_factory(), &arena)); + activation.InsertOrAssignValue("input", val); + + CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena, activation)); + if (!result.IsBool()) { + return absl::InvalidArgumentError( + "Predicate evaluate result must be a boolean value."); + } + return result.GetBool(); +} + +struct TestCase { + std::string name; + std::vector input_textprotos; + std::string expected_unparsed; + std::string eval_textproto; + bool expected_eval_result = true; + // If true, skip the eval step of the test. This is useful for tests where + // the expected expression does not share the same type structure as the + // input proto, such as empty messages. + bool skip_eval = false; +}; + +class ProtoToPredicateTest : public ::testing::TestWithParam {}; + +TEST_P(ProtoToPredicateTest, ConformanceTests) { + const TestCase& param = GetParam(); + + std::vector input_messages; + input_messages.reserve(param.input_textprotos.size()); + for (const auto& proto_str : param.input_textprotos) { + input_messages.push_back(ParseTestMessage(proto_str)); + } + + std::vector ptr_messages; + ptr_messages.reserve(input_messages.size()); + for (const auto& msg : input_messages) { + ptr_messages.push_back(&msg); + } + + absl::StatusOr ast_or; + if (input_messages.size() == 1) { + ast_or = ProtoToPredicateAst("input", input_messages[0]); + } else { + ast_or = ProtoToPredicateAst("input", absl::MakeSpan(ptr_messages)); + } + + ASSERT_THAT(ast_or, IsOk()); + cel::Ast ast = std::move(*ast_or); + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(auto unparsed, google::api::expr::Unparse(parsed_expr)); + + EXPECT_EQ(unparsed, param.expected_unparsed); + + if (!param.skip_eval) { + TestMessage eval_msg = ParseTestMessage(param.eval_textproto); + ASSERT_OK_AND_ASSIGN(bool eval_result, EvaluatePredicate(ast, eval_msg)); + EXPECT_EQ(eval_result, param.expected_eval_result); + } +} + +INSTANTIATE_TEST_SUITE_P( + ProtoToPredicateSubCases, ProtoToPredicateTest, + testing::Values( + TestCase{ + .name = "EmptyMessageTest", + .input_textprotos = {""}, + .expected_unparsed = "true", + .eval_textproto = "", + }, + TestCase{ + .name = "EmptyMessagesListTest", + .input_textprotos = {}, + .expected_unparsed = "true", + .eval_textproto = "", + }, + TestCase{ + .name = "PrimitivesTest", + .input_textprotos = {R"pb( + int32_value: 42 string_value: "hello" + )pb"}, + .expected_unparsed = + "input.int32_value == 42 && input.string_value == \"hello\"", + .eval_textproto = R"pb( + int32_value: 42 string_value: "hello" + )pb", + }, + TestCase{ + .name = "AllPrimitivesTest", + .input_textprotos = {R"pb( + int32_value: 42 + int64_value: 43 + uint32_value: 44 + uint64_value: 45 + float_value: 46.5 + double_value: 47.5 + bool_value: true + enum_value: TEST_ENUM_1 + string_value: "hello" + bytes_value: "world" + )pb"}, + .expected_unparsed = + "input.int32_value == 42 && input.int64_value == 43 && " + "input.uint32_value == 44u && input.uint64_value == 45u && " + "input.float_value == 46.5 && input.double_value == 47.5 && " + "input.string_value == \"hello\" && " + "input.bytes_value == b\"world\" && " + "input.bool_value == true && " + "input.enum_value == 1", + .eval_textproto = R"pb( + int32_value: 42 + int64_value: 43 + uint32_value: 44 + uint64_value: 45 + float_value: 46.5 + double_value: 47.5 + bool_value: true + enum_value: TEST_ENUM_1 + string_value: "hello" + bytes_value: "world" + )pb", + }, + TestCase{ + .name = "NestedMessageTest", + .input_textprotos = {R"pb( + message_value: { int32_value: 42 } + )pb"}, + .expected_unparsed = "input.message_value.int32_value == 42", + .eval_textproto = R"pb( + message_value: { int32_value: 42 } + )pb", + }, + TestCase{ + .name = "RepeatedFieldTest", + .input_textprotos = {R"pb( + int32_list: [ 1, 2 ] + )pb"}, + .expected_unparsed = + "1 in input.int32_list && 2 in input.int32_list", + .eval_textproto = R"pb( + int32_list: [ 1, 2 ] + )pb", + }, + TestCase{ + .name = "RepeatedFieldSingleElementTest", + .input_textprotos = {R"pb( + int32_list: [ 42 ] + )pb"}, + .expected_unparsed = "42 in input.int32_list", + .eval_textproto = R"pb( + int32_list: [ 42 ] + )pb", + }, + TestCase{ + .name = "RepeatedFieldEmptyTest", + .input_textprotos = {R"pb( + int32_list: [] + )pb"}, + .expected_unparsed = "true", + .eval_textproto = R"pb( + int32_list: [] + )pb", + }, + TestCase{ + .name = "ListFieldEvalNegative", + .input_textprotos = {R"pb( + int32_list: [ 1, 2 ] + )pb"}, + .expected_unparsed = + "1 in input.int32_list && 2 in input.int32_list", + .eval_textproto = R"pb( + int32_list: [ 1, 3 ] + )pb", + .expected_eval_result = false, + }, + TestCase{ + .name = "SingleRepeatedFieldAllPrimitivesTest", + .input_textprotos = {R"pb( + int32_list: [ 42 ] + int64_list: [ 43 ] + uint32_list: [ 44 ] + uint64_list: [ 45 ] + float_list: [ 46.5 ] + double_list: [ 47.5 ] + bool_list: [ true ] + enum_list: [ TEST_ENUM_1 ] + string_list: [ "hello" ] + bytes_list: [ "world" ] + )pb"}, + .expected_unparsed = "42 in input.int32_list && " + "43 in input.int64_list && " + "44u in input.uint32_list && " + "45u in input.uint64_list && " + "46.5 in input.float_list && " + "47.5 in input.double_list && " + "\"hello\" in input.string_list && " + "b\"world\" in input.bytes_list && " + "true in input.bool_list && " + "1 in input.enum_list", + .eval_textproto = R"pb( + int32_list: [ 42 ] + int64_list: [ 43 ] + uint32_list: [ 44 ] + uint64_list: [ 45 ] + float_list: [ 46.5 ] + double_list: [ 47.5 ] + bool_list: [ true ] + enum_list: [ TEST_ENUM_1 ] + string_list: [ "hello" ] + bytes_list: [ "world" ] + )pb", + }, + TestCase{ + .name = "MultipleRepeatedFieldAllPrimitivesTest", + .input_textprotos = {R"pb( + int32_list: [ 42, 142 ] + int64_list: [ 43, 143 ] + uint32_list: [ 44, 144 ] + uint64_list: [ 45, 145 ] + float_list: [ 46.5, 146.5 ] + double_list: [ 47.5, 147.5 ] + bool_list: [ true, false ] + enum_list: [ TEST_ENUM_1, TEST_ENUM_2 ] + string_list: [ "hello", "universe" ] + bytes_list: [ "world", "space" ] + )pb"}, + .expected_unparsed = + "42 in input.int32_list && 142 in input.int32_list && " + "43 in input.int64_list && 143 in input.int64_list && " + "44u in input.uint32_list && 144u in input.uint32_list && " + "45u in input.uint64_list && 145u in input.uint64_list && " + "46.5 in input.float_list && 146.5 in input.float_list && " + "47.5 in input.double_list && 147.5 in input.double_list && " + "\"hello\" in input.string_list && \"universe\" in " + "input.string_list && " + "b\"world\" in input.bytes_list && b\"space\" in " + "input.bytes_list && " + "true in input.bool_list && false in input.bool_list && " + "1 in input.enum_list && 2 in input.enum_list", + .eval_textproto = R"pb( + int32_list: [ 42, 142 ] + int64_list: [ 43, 143 ] + uint32_list: [ 44, 144 ] + uint64_list: [ 45, 145 ] + float_list: [ 46.5, 146.5 ] + double_list: [ 47.5, 147.5 ] + bool_list: [ true, false ] + enum_list: [ TEST_ENUM_1, TEST_ENUM_2 ] + string_list: [ "hello", "universe" ] + bytes_list: [ "world", "space" ] + )pb", + }, + TestCase{ + .name = "MapFieldTest", + .input_textprotos = {R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 2 } + )pb"}, + .expected_unparsed = "\"bar\" in input.string_int32_map && " + "input.string_int32_map[\"bar\"] == 2 && " + "\"foo\" in input.string_int32_map && " + "input.string_int32_map[\"foo\"] == 1", + .eval_textproto = R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 2 } + )pb", + }, + TestCase{ + .name = "MapFieldEvalNegativeVal", + .input_textprotos = {R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 2 } + )pb"}, + .expected_unparsed = "\"bar\" in input.string_int32_map && " + "input.string_int32_map[\"bar\"] == 2 && " + "\"foo\" in input.string_int32_map && " + "input.string_int32_map[\"foo\"] == 1", + .eval_textproto = R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 3 } + )pb", + .expected_eval_result = false, + }, + TestCase{ + .name = "MapFieldEvalNegativeNoKey", + .input_textprotos = {R"pb( + string_int32_map: { key: "foo" value: 1 } + string_int32_map: { key: "bar" value: 2 } + )pb"}, + .expected_unparsed = "\"bar\" in input.string_int32_map && " + "input.string_int32_map[\"bar\"] == 2 && " + "\"foo\" in input.string_int32_map && " + "input.string_int32_map[\"foo\"] == 1", + .eval_textproto = R"pb( + string_int32_map: { key: "foo" value: 1 } + )pb", + .expected_eval_result = false, + }, + TestCase{ + .name = "MapFieldIntKeySortingTest", + .input_textprotos = {R"pb( + int32_int32_map: { key: 10 value: 100 } + int32_int32_map: { key: 5 value: 50 } + int32_int32_map: { key: 8 value: 80 } + )pb"}, + .expected_unparsed = "5 in input.int32_int32_map && " + "input.int32_int32_map[5] == 50 && " + "8 in input.int32_int32_map && " + "input.int32_int32_map[8] == 80 && " + "10 in input.int32_int32_map && " + "input.int32_int32_map[10] == 100", + .eval_textproto = R"pb( + int32_int32_map: { key: 5 value: 50 } + int32_int32_map: { key: 8 value: 80 } + int32_int32_map: { key: 10 value: 100 } + )pb", + }, + TestCase{ + .name = "MultipleMessagesTest", + .input_textprotos = {R"pb( + int32_value: 42 + )pb", + R"pb( + int32_value: 41 string_value: "hello" + )pb"}, + .expected_unparsed = + "input.int32_value == 42 || input.int32_value == 41 && " + "input.string_value == \"hello\"", + .eval_textproto = R"pb( + int32_value: 41 string_value: "hello" + )pb", + }, + TestCase{ + .name = "RepeatedMessageFieldTest", + .input_textprotos = {R"pb( + message_list: + [ { int32_value: 42 } + , { int32_value: 43 }] + )pb"}, + .expected_unparsed = "input.message_list.int32_value == 42 || " + "input.message_list.int32_value == 43", + .skip_eval = true, + }, + TestCase{ + .name = "RepeatedMessageSingleElementTest", + .input_textprotos = {R"pb( + message_list: + [ { int32_value: 42 }] + )pb"}, + .expected_unparsed = "input.message_list.int32_value == 42", + .skip_eval = true, + })); + +struct PolicyTestCase { + std::string name; + std::string json_input; + std::string expected_unparsed; +}; + +class PolicyJsonTest : public ::testing::TestWithParam {}; + +TEST_P(PolicyJsonTest, Conformance) { + const PolicyTestCase& param = GetParam(); + + cel::cpp::tools::Policy policy; + google::protobuf::json::ParseOptions options; + options.ignore_unknown_fields = true; + auto status = + google::protobuf::json::JsonStringToMessage(param.json_input, &policy, options); + ASSERT_THAT(status, IsOk()) << "Failed to parse JSON: " << param.json_input; + + absl::StatusOr ast_or; + std::vector ptr_messages; + ptr_messages.reserve(policy.destinations_size()); + for (const auto& dest : policy.destinations()) { + ptr_messages.push_back(&dest); + } + + if (ptr_messages.empty()) { + auto parsed_expr_or = google::api::expr::parser::Parse("false"); + ASSERT_THAT(parsed_expr_or, IsOk()); + auto ast_ptr_or = cel::CreateAstFromParsedExpr(*parsed_expr_or); + ASSERT_THAT(ast_ptr_or, IsOk()); + ast_or = std::move(**ast_ptr_or); + } else if (ptr_messages.size() == 1) { + ast_or = ProtoToPredicateAst("dest", *ptr_messages[0]); + } else { + ast_or = ProtoToPredicateAst("dest", absl::MakeSpan(ptr_messages)); + } + + ASSERT_THAT(ast_or, IsOk()); + cel::Ast ast = std::move(*ast_or); + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(auto unparsed, google::api::expr::Unparse(parsed_expr)); + + EXPECT_EQ(unparsed, param.expected_unparsed); +} + +INSTANTIATE_TEST_SUITE_P( + PolicyJsonSubCases, PolicyJsonTest, + testing::Values( + PolicyTestCase{ + .name = "SimpleMatch", + .json_input = + R"({ "destinations": [ { "agent": { "id": "agent-007" } } ] })", + .expected_unparsed = "dest.agent.name == \"agent-007\"", + }, + PolicyTestCase{ + .name = "MultipleFields", + .json_input = + R"({ "destinations": [ { + "tool": { + "name": "admin_tool", + "annotations": { + "read_only_hint": false + } + } + } + ] })", + .expected_unparsed = + "dest.tool.name == \"admin_tool\" && " + "dest.tool.annotations.read_only_hint == false", + }, + PolicyTestCase{ + .name = "RepeatedMessages", + .json_input = + R"({ "destinations": [ + { "agent": { "id": "worker-1" } }, + { "agent": { "id": "worker-2" } }, + ] })", + .expected_unparsed = "dest.agent.name == \"worker-1\" || " + "dest.agent.name == \"worker-2\"", + }, + PolicyTestCase{ + .name = "RepeatedPrimitiveArraySingleElement", + .json_input = + R"({ "destinations": [ { + "tool": { + "role_members": { + "admin": { + "principals": ["alice"] + } + } + } + } ] })", + .expected_unparsed = + "\"admin\" in dest.tool.role_members && " + "\"alice\" in dest.tool.role_members[\"admin\"].principals", + }, + PolicyTestCase{ + .name = "RepeatedArrayEmpty", + .json_input = R"({ "destinations": [ { "tool": { } } ] })", + .expected_unparsed = "true", + }, + PolicyTestCase{ + .name = "MapEquality", + .json_input = + R"({ "destinations": [ + { "tool": { + "name": "shell", + "labels": { + "cluster": "us-central1", + "project": "dev" + } + } + } ] })", + .expected_unparsed = + "dest.tool.name == \"shell\" && \"cluster\" in " + "dest.tool.labels && dest.tool.labels[\"cluster\"] == " + "\"us-central1\" && \"project\" in dest.tool.labels && " + "dest.tool.labels[\"project\"] == \"dev\"", + }, + PolicyTestCase{ + .name = "NestedMapEquality", + .json_input = + R"({ "destinations": [ + { "tool": { + "role_members": { + "admin": { + "all_users": true + } + } + } } + ] })", + .expected_unparsed = + "\"admin\" in dest.tool.role_members && " + "dest.tool.role_members[\"admin\"].all_users == true", + }, + PolicyTestCase{ + .name = "EmptyPolicy", + .json_input = "{}", + .expected_unparsed = "false", + })); + +} // namespace +} // namespace cel::tools diff --git a/tools/testdata/BUILD b/tools/testdata/BUILD index 493f0ff2f..c88c9c478 100644 --- a/tools/testdata/BUILD +++ b/tools/testdata/BUILD @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -load( - "@com_github_google_flatbuffers//:build_defs.bzl", - "flatbuffer_library_public", -) +load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public") +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") licenses(["notice"]) @@ -46,3 +45,15 @@ cc_library( linkstatic = True, deps = ["@com_github_google_flatbuffers//:runtime_cc"], ) + +proto_library( + name = "test_policy_proto", + srcs = ["test_policy.proto"], + visibility = ["//tools:__subpackages__"], +) + +cc_proto_library( + name = "test_policy_cc_proto", + visibility = ["//tools:__subpackages__"], + deps = [":test_policy_proto"], +) diff --git a/tools/testdata/test_policy.proto b/tools/testdata/test_policy.proto new file mode 100644 index 000000000..b5d424c04 --- /dev/null +++ b/tools/testdata/test_policy.proto @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Test schema representing client-configured policies. +// It is used by the `proto_to_predicate` tool to translate Protobuf policies +// into CEL predicates. +edition = "2023"; + +package cel.cpp.tools; + +option cc_enable_arenas = true; + +// Represents the targeted client agent. +message Agent { + string name = 1 [json_name = "id"]; +} + +// Specifies additional metadata tool annotations. +message ToolAnnotations { + bool read_only_hint = 1; +} + +// Represents a mapped nested message entry value inside map fields. +message Members { + repeated string principals = 1; + + repeated string regions = 2; + + bool all_users = 3; + + bool all_authenticated_users = 4; +} + +// Represents a metadata tool block. +message Tool { + // The name of the tool. + string name = 1; + + // Additional metadata annotations for the tool. + ToolAnnotations annotations = 2; + + // A string-to-string map, transpiled as conjoined existence and equality + // checks. + map labels = 3; + + // A map with string keys representing roles and Member instances as values. + map role_members = 4; +} + +// Represents a policy mapping destination block. +message Target { + oneof kind { + Agent agent = 1; + Tool tool = 2; + } +} + +// Represents the top-level policy containing multiple alternate destination +// rules. +message Policy { + repeated Target destinations = 1; +} From 9597d49ef616726bdc8b2a553fd89fd46ad8b493 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 15 Jun 2026 11:43:00 -0700 Subject: [PATCH 065/350] Add support for context types env.yaml Add support for declaring a protobuf context message whose top level fields are declared as variables in the CEL environment. PiperOrigin-RevId: 932580314 --- env/BUILD | 1 + env/config.h | 6 ++++++ env/env.cc | 5 +++++ env/env_test.cc | 19 +++++++++++++++++++ env/env_yaml.cc | 30 ++++++++++++++++++++++++++++++ env/env_yaml_test.cc | 41 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 102 insertions(+) diff --git a/env/BUILD b/env/BUILD index 3035e11ac..bd82e8ec6 100644 --- a/env/BUILD +++ b/env/BUILD @@ -130,6 +130,7 @@ cc_library( "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/env/config.h b/env/config.h index e427832ff..68e4a1dd9 100644 --- a/env/config.h +++ b/env/config.h @@ -32,6 +32,11 @@ class Config { void SetName(std::string name) { name_ = std::move(name); } std::string GetName() const { return name_; } + void SetContextType(std::string context_type) { + context_type_ = std::move(context_type); + } + std::string GetContextType() const { return context_type_; } + struct ContainerConfig { std::string name; std::vector abbreviations; @@ -150,6 +155,7 @@ class Config { private: std::string name_; + std::string context_type_; ContainerConfig container_config_; std::vector extension_configs_; StandardLibraryConfig standard_library_config_; diff --git a/env/env.cc b/env/env.cc index 6cd3a3cdc..22d24295e 100644 --- a/env/env.cc +++ b/env/env.cc @@ -138,6 +138,11 @@ absl::StatusOr> Env::NewCompilerBuilder() { for (const auto& abbr : config_.GetContainerConfig().abbreviations) { CEL_RETURN_IF_ERROR(container.AddAbbreviation(abbr)); } + + if (!config_.GetContextType().empty()) { + CEL_RETURN_IF_ERROR( + checker_builder.AddContextDeclaration(config_.GetContextType())); + } for (const auto& alias : config_.GetContainerConfig().aliases) { CEL_RETURN_IF_ERROR(container.AddAlias(alias.alias, alias.qualified_name)); } diff --git a/env/env_test.cc b/env/env_test.cc index b599aa569..fda87dfab 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -344,6 +344,25 @@ TEST(ContainerConfigTest, ContainerConfigWithAliases) { EXPECT_THAT(result.GetIssues(), IsEmpty()) << result.FormatError(); } +TEST(ContextVariableConfigTest, Basic) { + Env env; + env.SetDescriptorPool(internal::GetSharedTestingDescriptorPool()); + Config config; + config.SetContextType("cel.expr.conformance.proto3.TestAllTypes"); + env.SetConfig(config); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, env.NewCompiler()); + + // Top-level fields of TestAllTypes like "single_int32" should resolve + // successfully. + ASSERT_OK_AND_ASSIGN(auto result, compiler->Compile("single_int32 > 10")); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto result_invalid, + compiler->Compile("non_existent_field > 10")); + EXPECT_THAT(result_invalid.GetIssues(), Not(IsEmpty())); +} + struct VariableConfigWithValueTestCase { Config::VariableConfig variable_config; std::string validate_type_expr; diff --git a/env/env_yaml.cc b/env/env_yaml.cc index 1bbfe6b36..e7b8a7885 100644 --- a/env/env_yaml.cc +++ b/env/env_yaml.cc @@ -26,6 +26,7 @@ #include "absl/base/no_destructor.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/escaping.h" @@ -1245,6 +1246,34 @@ void EmitFunctionConfigs(const Config& env_config, YAML::Emitter& out, } out << YAML::EndSeq; } + +absl::Status ParseContextVariableConfig(Config& config, absl::string_view yaml, + const YAML::Node& root) { + const YAML::Node context_variable = root["context_variable"]; + if (!context_variable.IsDefined()) { + return absl::OkStatus(); + } + if (!context_variable.IsMap()) { + return YamlError(yaml, context_variable, + "Node 'context_variable' is not a map"); + } + + const YAML::Node type_name = context_variable["type_name"]; + const YAML::Node type = context_variable["type"]; + const YAML::Node* type_node = nullptr; + if (type.IsDefined() && type.IsScalar()) { + type_node = &type; + } else if (type_name.IsDefined() && type_name.IsScalar()) { + type_node = &type_name; + } else { + return YamlError(yaml, context_variable, + "Node 'context_variable' does not have a valid type"); + } + ABSL_DCHECK(type_node != nullptr); + config.SetContextType(GetString(yaml, *type_node)); + return absl::OkStatus(); +} + } // namespace absl::StatusOr EnvConfigFromYaml(const std::string& yaml) { @@ -1263,6 +1292,7 @@ absl::StatusOr EnvConfigFromYaml(const std::string& yaml) { CEL_RETURN_IF_ERROR(ParseContainerConfig(config, yaml, root)); CEL_RETURN_IF_ERROR(ParseExtensionConfigs(config, yaml, root)); CEL_RETURN_IF_ERROR(ParseStandardLibraryConfig(config, yaml, root)); + CEL_RETURN_IF_ERROR(ParseContextVariableConfig(config, yaml, root)); CEL_RETURN_IF_ERROR(ParseVariableConfigs(config, yaml, root)); CEL_RETURN_IF_ERROR(ParseFunctionConfigs(config, yaml, root)); return config; diff --git a/env/env_yaml_test.cc b/env/env_yaml_test.cc index a60048617..9c5b3f04f 100644 --- a/env/env_yaml_test.cc +++ b/env/env_yaml_test.cc @@ -216,6 +216,47 @@ TEST(EnvYamlTest, ParseVariableConfigWithTypeParams) { EXPECT_THAT(type_info.params[1].params, IsEmpty()); } +TEST(EnvYamlTest, ParseContextVariableConfig) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + context_variable: + type_name: "cel.expr.conformance.proto3.TestAllTypes" + )yaml")); + + EXPECT_EQ(config.GetContextType(), + "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(EnvYamlTest, ParseContextVariableConfigAlternativeSyntax) { + ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( + context_variable: + type: "cel.expr.conformance.proto3.TestAllTypes" + )yaml")); + + EXPECT_EQ(config.GetContextType(), + "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(EnvYamlTest, ParseContextVariableMalformedContextVariable) { + EXPECT_THAT(EnvConfigFromYaml(R"yaml( + context_variable: 123 + + )yaml"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Node 'context_variable' is not a map"))); +} + +TEST(EnvYamlTest, ParseContextVariableMalformedContextVariable2) { + EXPECT_THAT( + EnvConfigFromYaml(R"yaml( + context_variable: + type: + foo: bar + )yaml"), + StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr("Node 'context_variable' does not have a valid type"))); +} + TEST(EnvYamlTest, ParseVariableConfigWithTypeParamsLegacySyntax) { ASSERT_OK_AND_ASSIGN(Config config, EnvConfigFromYaml(R"yaml( variables: From 01e6b6379d8100f6f6dcb6e41627037d1b2a7112 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 16 Jun 2026 06:44:51 -0700 Subject: [PATCH 066/350] Export policy compiler. Co-authored-by: Dmitri Plotnikov PiperOrigin-RevId: 933062225 --- MODULE.bazel | 14 + conformance/policy/BUILD | 78 ++ .../policy/policy_conformance_test.bzl | 46 + conformance/policy/policy_conformance_test.cc | 659 ++++++++++ internal/BUILD | 1 + internal/runfiles.cc | 15 +- internal/runfiles.h | 6 + policy/BUILD | 239 ++++ policy/cel_policy.cc | 273 +++++ policy/cel_policy.h | 320 +++++ policy/cel_policy_parse_context.cc | 49 + policy/cel_policy_parse_context.h | 65 + policy/cel_policy_parse_result.cc | 91 ++ policy/cel_policy_parse_result.h | 105 ++ policy/cel_policy_parser.h | 40 + policy/cel_policy_test.cc | 220 ++++ policy/cel_policy_validation_result.cc | 32 + policy/cel_policy_validation_result.h | 84 ++ policy/compiler.cc | 1058 +++++++++++++++++ policy/compiler.h | 50 + policy/compiler_test.cc | 946 +++++++++++++++ policy/internal/BUILD | 68 ++ policy/internal/issue_reporter.cc | 45 + policy/internal/issue_reporter.h | 57 + policy/internal/optimizer_expr_factory.cc | 373 ++++++ policy/internal/optimizer_expr_factory.h | 419 +++++++ .../internal/optimizer_expr_factory_test.cc | 570 +++++++++ policy/test_custom_yaml_policy_parser.cc | 188 +++ policy/test_util.cc | 221 ++++ policy/test_util.h | 33 + policy/testdata/BUILD | 19 + policy/testdata/cel_policy.yaml | 42 + policy/testdata/cel_policy_parser.baseline | 89 ++ policy/testdata/custom_policy_format.yaml | 29 + .../custom_policy_format_parser.baseline | 75 ++ .../custom_policy_format_with_errors.yaml | 33 + ..._policy_format_with_errors_parser.baseline | 16 + policy/testdata/nested_rule.yaml | 37 + policy/testdata/nested_rule_parser.baseline | 84 ++ policy/yaml_policy_parser.cc | 411 +++++++ policy/yaml_policy_parser.h | 135 +++ policy/yaml_policy_parser_test.cc | 305 +++++ 42 files changed, 7639 insertions(+), 1 deletion(-) create mode 100644 conformance/policy/BUILD create mode 100644 conformance/policy/policy_conformance_test.bzl create mode 100644 conformance/policy/policy_conformance_test.cc create mode 100644 policy/BUILD create mode 100644 policy/cel_policy.cc create mode 100644 policy/cel_policy.h create mode 100644 policy/cel_policy_parse_context.cc create mode 100644 policy/cel_policy_parse_context.h create mode 100644 policy/cel_policy_parse_result.cc create mode 100644 policy/cel_policy_parse_result.h create mode 100644 policy/cel_policy_parser.h create mode 100644 policy/cel_policy_test.cc create mode 100644 policy/cel_policy_validation_result.cc create mode 100644 policy/cel_policy_validation_result.h create mode 100644 policy/compiler.cc create mode 100644 policy/compiler.h create mode 100644 policy/compiler_test.cc create mode 100644 policy/internal/BUILD create mode 100644 policy/internal/issue_reporter.cc create mode 100644 policy/internal/issue_reporter.h create mode 100644 policy/internal/optimizer_expr_factory.cc create mode 100644 policy/internal/optimizer_expr_factory.h create mode 100644 policy/internal/optimizer_expr_factory_test.cc create mode 100644 policy/test_custom_yaml_policy_parser.cc create mode 100644 policy/test_util.cc create mode 100644 policy/test_util.h create mode 100644 policy/testdata/BUILD create mode 100644 policy/testdata/cel_policy.yaml create mode 100644 policy/testdata/cel_policy_parser.baseline create mode 100644 policy/testdata/custom_policy_format.yaml create mode 100644 policy/testdata/custom_policy_format_parser.baseline create mode 100644 policy/testdata/custom_policy_format_with_errors.yaml create mode 100644 policy/testdata/custom_policy_format_with_errors_parser.baseline create mode 100644 policy/testdata/nested_rule.yaml create mode 100644 policy/testdata/nested_rule_parser.baseline create mode 100644 policy/yaml_policy_parser.cc create mode 100644 policy/yaml_policy_parser.h create mode 100644 policy/yaml_policy_parser_test.cc diff --git a/MODULE.bazel b/MODULE.bazel index 43d0485d2..187d68164 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,6 +31,7 @@ bazel_dep( name = "rules_python", version = "1.6.3", ) +bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep( name = "protobuf", version = "34.1", @@ -96,3 +97,16 @@ bazel_dep( name = "yaml-cpp", version = "0.9.0", ) + +_CEL_POLICY_TAG = "ebfb2361f47080af643c14cf4da4c2b551a68740" + +_CEL_POLICY_SHA = "ea69e9c6b7bd5bc37d358148aebd2fcca38bc7c45a23feb635de72338e0327c1" + +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "cel_policy", + sha256 = _CEL_POLICY_SHA, + strip_prefix = "cel-policy-%s" % _CEL_POLICY_TAG, + url = "https://github.com/cel-expr/cel-policy/archive/%s.tar.gz" % _CEL_POLICY_TAG, +) diff --git a/conformance/policy/BUILD b/conformance/policy/BUILD new file mode 100644 index 000000000..29210e02d --- /dev/null +++ b/conformance/policy/BUILD @@ -0,0 +1,78 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load( + "//conformance/policy:policy_conformance_test.bzl", + "cel_policy_conformance_test", +) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "policy_conformance_test_lib", + testonly = True, + srcs = ["policy_conformance_test.cc"], + deps = [ + "//common:ast", + "//common:source", + "//common:value", + "//common/internal:value_conversion", + "//compiler", + "//env", + "//env:config", + "//env:env_runtime", + "//env:env_std_extensions", + "//env:env_yaml", + "//env:runtime_std_extensions", + "//extensions/protobuf:bind_proto_to_activation", + "//extensions/protobuf:enum_adapter", + "//internal:runfiles", + "//internal:status_macros", + "//internal:testing_descriptor_pool", + "//internal:testing_no_main", + "//policy:cel_policy", + "//policy:cel_policy_parser", + "//policy:cel_policy_validation_result", + "//policy:compiler", + "//policy:test_util", + "//policy:yaml_policy_parser", + "//runtime", + "//runtime:activation", + "//runtime:function_adapter", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_cel_spec//proto/cel/expr:value_cc_proto", + "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", + "@com_google_protobuf//:protobuf", + ], + alwayslink = True, +) + +cel_policy_conformance_test( + name = "policy_conformance_test", + example = "@cel_policy//conformance:testdata/nested_rule/policy.yaml", + skip_tests = [ + # TODO(b/506179116): Fix these. + # Need to add k8s custom yaml parser and mock runtime. + "k8s", + ], + test_files = [ + "@cel_policy//conformance:testdata", + ], +) diff --git a/conformance/policy/policy_conformance_test.bzl b/conformance/policy/policy_conformance_test.bzl new file mode 100644 index 000000000..0b4d1a4c6 --- /dev/null +++ b/conformance/policy/policy_conformance_test.bzl @@ -0,0 +1,46 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains build rules for generating policy conformance test targets. +""" + +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +def cel_policy_conformance_test(name, test_files, example, skip_tests = [], **kwargs): + """Generates a policy conformance test target. + + Args: + name: Name of the test target. + test_files: List of targets or files representing the test data. + example: A specific example file from test_files used for runfiles resolution. + skip_tests: List of test cases to skip. + testdata_dir: Path to testdata directory under runfiles. + **kwargs: Additional arguments passed to the underlying cc_test. + """ + args = ["--gunit_fail_if_no_test_linked"] + args.append("--testdata_example='$(rlocationpath {})'".format(example)) + + if skip_tests: + args.append("--skip_tests=" + ",".join(skip_tests)) + + cc_test( + name = name, + data = test_files + [example], + deps = [ + "//conformance/policy:policy_conformance_test_lib", + ], + args = args, + **kwargs + ) diff --git a/conformance/policy/policy_conformance_test.cc b/conformance/policy/policy_conformance_test.cc new file mode 100644 index 000000000..0d68f8abf --- /dev/null +++ b/conformance/policy/policy_conformance_test.cc @@ -0,0 +1,659 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +// NOLINTNEXTLINE(build/c++17) for OSS compatibility +#include + +#include "cel/expr/eval.pb.h" +#include "absl/flags/flag.h" +#include "absl/log/absl_check.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/internal/value_conversion.h" +#include "common/source.h" +#include "common/value.h" +#include "compiler/compiler.h" +#include "env/config.h" +#include "env/env.h" +#include "env/env_runtime.h" +#include "env/env_std_extensions.h" +#include "env/env_yaml.h" +#include "env/runtime_std_extensions.h" +#include "extensions/protobuf/bind_proto_to_activation.h" +#include "extensions/protobuf/enum_adapter.h" +#include "internal/runfiles.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_validation_result.h" +#include "policy/compiler.h" +#include "policy/test_util.h" +#include "policy/yaml_policy_parser.h" +#include "runtime/activation.h" +#include "runtime/function_adapter.h" +#include "runtime/runtime.h" +#include "cel/expr/conformance/test/suite.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/dynamic_message.h" +#include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" + +// Use a specific file to handle bazel runfiles resolution correctly. We find +// parent directory named 'testdata' to use as the root of the test cases. +ABSL_FLAG(std::string, testdata_example, "", + "Path to a specific example file."); +ABSL_FLAG(std::vector, skip_tests, {}, + "Comma-separated list of tests to skip."); + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::cel::expr::conformance::test::TestSuite; +using ::cel::internal::GetSharedTestingDescriptorPool; +using ::testing::HasSubstr; + +// Implementations for extension functions referenced in conformance tests. +cel::Value LocationCode(const cel::StringValue& ip, + const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* factory, google::protobuf::Arena* arena) { + std::string ip_str = ip.ToString(); + if (ip_str == "10.0.0.1") return cel::StringValue(arena, "us"); + if (ip_str == "10.0.0.2") return cel::StringValue(arena, "de"); + return cel::StringValue(arena, "ir"); +} + +// TODO(uncreated-issue/92): This should be migrated to use the testrunner utility +// after adding support for reading the yaml specification for envs/tests. +class InputEvaluator { + public: + static absl::StatusOr> Create( + const std::shared_ptr& pool) { + cel::Env env; + env.SetDescriptorPool(pool); + cel::RegisterStandardExtensions(env); + + cel::EnvRuntime env_runtime; + env_runtime.SetDescriptorPool(pool); + cel::RegisterStandardExtensions(env_runtime); + env_runtime.mutable_runtime_options().enable_qualified_type_identifiers = + true; + + // Enable default extensions (optional, bindings) + cel::Config config; + CEL_RETURN_IF_ERROR(config.AddExtensionConfig( + "optional", cel::Config::ExtensionConfig::kLatest)); + CEL_RETURN_IF_ERROR(config.AddExtensionConfig( + "bindings", cel::Config::ExtensionConfig::kLatest)); + env.SetConfig(config); + env_runtime.SetConfig(config); + + auto compiler_builder_or = env.NewCompilerBuilder(); + CEL_ASSIGN_OR_RETURN(auto compiler_builder, std::move(compiler_builder_or)); + compiler_builder->GetParserBuilder().GetOptions().enable_optional_syntax = + true; + CEL_ASSIGN_OR_RETURN(auto compiler, compiler_builder->Build()); + + auto runtime_builder_or = env_runtime.CreateRuntimeBuilder(); + CEL_ASSIGN_OR_RETURN(auto runtime_builder, std::move(runtime_builder_or)); + + // Register conformance enums + for (const auto& enum_name : + {"cel.expr.conformance.proto2.GlobalEnum", + "cel.expr.conformance.proto3.GlobalEnum", + "cel.expr.conformance.proto2.TestAllTypes.NestedEnum", + "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"}) { + auto* enum_desc = pool->FindEnumTypeByName(enum_name); + if (enum_desc != nullptr) { + CEL_RETURN_IF_ERROR(cel::extensions::RegisterProtobufEnum( + runtime_builder.type_registry(), enum_desc)); + } + } + + CEL_ASSIGN_OR_RETURN(auto runtime, std::move(runtime_builder).Build()); + + return absl::WrapUnique( + new InputEvaluator(std::move(compiler), std::move(runtime))); + } + + absl::StatusOr Evaluate( + absl::string_view expr_str, google::protobuf::Arena* arena, + google::protobuf::MessageFactory* message_factory) const { + CEL_ASSIGN_OR_RETURN(auto validation_result, compiler_->Compile(expr_str)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to compile input expr: ", expr_str)); + } + CEL_ASSIGN_OR_RETURN(auto ast, validation_result.ReleaseAst()); + CEL_ASSIGN_OR_RETURN( + auto program, + runtime_->CreateProgram(std::make_unique(std::move(*ast)))); + cel::Activation activation; + EvaluateOptions options; + options.message_factory = message_factory; + return program->Evaluate(arena, activation, options); + } + + private: + InputEvaluator(std::unique_ptr compiler, + std::unique_ptr runtime) + : compiler_(std::move(compiler)), runtime_(std::move(runtime)) {} + + std::unique_ptr compiler_; + std::unique_ptr runtime_; +}; + +absl::StatusOr EvaluateInputValue( + const cel::expr::conformance::test::InputValue& input_val, + const InputEvaluator& evaluator, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena) { + if (input_val.has_expr()) { + return evaluator.Evaluate(input_val.expr(), arena, message_factory); + } + if (input_val.has_value()) { + return cel::test::FromExprValue(input_val.value(), descriptor_pool, + message_factory, arena); + } + return absl::InvalidArgumentError("Empty InputValue"); +} + +class CelValueMatcherImpl + : public testing::MatcherInterface { + public: + CelValueMatcherImpl(cel::Value expected_val, + const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena) + : expected_val_(std::move(expected_val)), + pool_(pool), + message_factory_(message_factory), + arena_(arena) {} + + bool MatchAndExplain(const cel::Value& actual_val, + testing::MatchResultListener* listener) const override { + cel::Value actual = actual_val; + if (actual.IsOptional() && !expected_val_.IsOptional()) { + auto opt_val = actual.AsOptional(); + if (opt_val->HasValue()) { + actual = opt_val->Value(); + } + } + cel::Value eq_result; + auto eq_status = actual.Equal(expected_val_, pool_, message_factory_, + arena_, &eq_result); + if (!eq_status.ok()) { + *listener << "equality check failed with status: " << eq_status; + return false; + } + if (!eq_result.IsTrue()) { + *listener << "expected: " << expected_val_.DebugString() + << "\nactual: " << actual.DebugString(); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os) const override { + *os << "is equal to " << expected_val_.DebugString(); + } + + void DescribeNegationTo(std::ostream* os) const override { + *os << "is not equal to " << expected_val_.DebugString(); + } + + private: + cel::Value expected_val_; + const google::protobuf::DescriptorPool* pool_; + google::protobuf::MessageFactory* message_factory_; + google::protobuf::Arena* arena_; +}; + +absl::StatusOr> MakeExpectedValueMatcher( + const cel::expr::conformance::test::TestOutput& output, + const InputEvaluator& input_evaluator, const google::protobuf::DescriptorPool* pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena) { + cel::Value expected_val; + if (output.has_result_expr()) { + CEL_ASSIGN_OR_RETURN( + expected_val, + input_evaluator.Evaluate(output.result_expr(), arena, message_factory)); + } else if (output.has_result_value()) { + CEL_ASSIGN_OR_RETURN(expected_val, + cel::test::FromExprValue(output.result_value(), pool, + message_factory, arena)); + } else { + return absl::InvalidArgumentError("Unsupported output kind"); + } + return testing::Matcher( + new CelValueMatcherImpl(expected_val, pool, message_factory, arena)); +} + +bool ShouldRunTest(absl::string_view test_name, + const std::vector& skip_tests) { + for (const std::string& skip : skip_tests) { + if (absl::StartsWith(test_name, skip)) { + return false; + } + } + return true; +} + +absl::Status PopulateActivation( + const cel::expr::conformance::test::TestCase& test, + const InputEvaluator& input_evaluator, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + absl::string_view context_msg_type_name, google::protobuf::Arena* arena, + Activation& activation) { + if (!test.has_input_context()) { + for (const auto& [var_name, input_val] : test.input()) { + CEL_ASSIGN_OR_RETURN( + auto val, + EvaluateInputValue(input_val, input_evaluator, descriptor_pool, + message_factory, arena)); + activation.InsertOrAssignValue(var_name, std::move(val)); + } + return absl::OkStatus(); + } + + const auto& input_context = test.input_context(); + const google::protobuf::Message* context_message = nullptr; + + if (input_context.has_context_message()) { + const google::protobuf::Any& any_msg = input_context.context_message(); + const google::protobuf::Descriptor* msg_descriptor = + descriptor_pool->FindMessageTypeByName(context_msg_type_name); + if (msg_descriptor == nullptr) { + return absl::NotFoundError(absl::StrCat( + "Failed to find message descriptor for: ", context_msg_type_name)); + } + const google::protobuf::Message* prototype = + message_factory->GetPrototype(msg_descriptor); + if (prototype == nullptr) { + return absl::NotFoundError( + absl::StrCat("Failed to get prototype for: ", context_msg_type_name)); + } + auto* buf = prototype->New(arena); + if (!any_msg.UnpackTo(buf)) { + return absl::InvalidArgumentError(absl::StrCat( + "Failed to unpack context message to ", context_msg_type_name)); + } + context_message = buf; + } else if (input_context.has_context_expr() && + !context_msg_type_name.empty()) { + CEL_ASSIGN_OR_RETURN(cel::Value evaluated_val, + input_evaluator.Evaluate(input_context.context_expr(), + arena, message_factory)); + + if (!evaluated_val.IsParsedMessage()) { + return absl::InvalidArgumentError( + absl::StrCat("Context expression did not evaluate to a message: ", + input_context.context_expr())); + } + if (evaluated_val.GetParsedMessage().GetDescriptor()->full_name() != + context_msg_type_name) { + return absl::InvalidArgumentError(absl::StrCat( + "Context expression evaluated to a message of type ", + evaluated_val.GetParsedMessage().GetDescriptor()->full_name(), + " which does not match the expected type ", context_msg_type_name)); + } + context_message = static_cast( + evaluated_val.GetParsedMessage().operator->()); + } + if (context_message == nullptr) { + return absl::InvalidArgumentError( + "Failed to resolve context message for test case"); + } + + return cel::extensions::BindProtoToActivation( + *context_message, + cel::extensions::BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool, message_factory, arena, &activation); +} + +class PolicyTestSuiteRunner { + public: + PolicyTestSuiteRunner(std::string suite_name, + std::unique_ptr compiler, + std::unique_ptr runtime, + std::shared_ptr policy_source, + CelPolicyValidationResult compile_result, + std::shared_ptr pool, + std::shared_ptr message_factory, + std::shared_ptr input_evaluator, + std::string context_msg_type_name, + bool expect_compile_fail = false) + : suite_name_(std::move(suite_name)), + compiler_(std::move(compiler)), + runtime_(std::move(runtime)), + policy_source_(std::move(policy_source)), + compile_result_(std::move(compile_result)), + pool_(std::move(pool)), + message_factory_(std::move(message_factory)), + input_evaluator_(std::move(input_evaluator)), + context_msg_type_name_(std::move(context_msg_type_name)), + expect_compile_fail_(expect_compile_fail) {} + + void RunTest(const cel::expr::conformance::test::TestCase& test, + absl::string_view full_test_name) { + const auto& output = test.output(); + + if (expect_compile_fail_) { + ASSERT_FALSE(compile_result_.IsValid()) + << "Expected compilation to fail in " << full_test_name; + ASSERT_TRUE(output.has_eval_error()) + << "Expected eval_error to be present in compile error test " + << full_test_name; + std::string err_msg = compile_result_.FormatIssues(); + for (const auto& expected_err : output.eval_error().errors()) { + EXPECT_THAT(err_msg, HasSubstr(expected_err.message())) + << "Did not find expected compile time error"; + } + return; + } + + // Compilation should have succeeded for evaluation tests + ASSERT_TRUE(compile_result_.IsValid()) + << "Compilation has validation errors in " << full_test_name << ": " + << compile_result_.FormatIssues(); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime_->CreateProgram(std::make_unique( + *compile_result_.GetAst()))); + + // Parse Inputs and evaluate them + google::protobuf::Arena arena; + Activation activation; + ASSERT_THAT(PopulateActivation(test, *input_evaluator_, pool_.get(), + message_factory_.get(), + context_msg_type_name_, &arena, activation), + IsOk()); + + // Evaluate Policy + auto eval_result_or = program->Evaluate(&arena, activation); + ASSERT_THAT(eval_result_or.status(), IsOk()) + << "Evaluation failed in " << full_test_name; + cel::Value actual_val = *eval_result_or; + + ASSERT_OK_AND_ASSIGN( + auto matcher, + MakeExpectedValueMatcher(output, *input_evaluator_, pool_.get(), + message_factory_.get(), &arena)); + + // Apply matcher to the output of evaluation + EXPECT_THAT(actual_val, matcher) << "Test failed: " << full_test_name; + } + + private: + std::string suite_name_; + std::unique_ptr compiler_; + std::unique_ptr runtime_; + std::shared_ptr policy_source_; + CelPolicyValidationResult compile_result_; + std::shared_ptr pool_; + std::shared_ptr message_factory_; + std::shared_ptr input_evaluator_; + std::string context_msg_type_name_; + bool expect_compile_fail_; +}; + +class CelPolicyTest : public testing::Test { + public: + explicit CelPolicyTest(std::shared_ptr runner, + cel::expr::conformance::test::TestCase test_case, + std::string full_test_name, bool skip) + : runner_(std::move(runner)), + test_case_(std::move(test_case)), + full_test_name_(std::move(full_test_name)), + skip_(skip) {} + + void TestBody() override { + if (skip_) { + GTEST_SKIP() << "Skipping test: " << full_test_name_; + } + EXPECT_NO_FATAL_FAILURE(runner_->RunTest(test_case_, full_test_name_)); + } + + private: + std::shared_ptr runner_; + cel::expr::conformance::test::TestCase test_case_; + std::string full_test_name_; + bool skip_; +}; + + +absl::Status RegisterTestSuite( + const std::filesystem::path& dir_path, const std::string& suite_name, + const std::shared_ptr& input_evaluator, + const std::shared_ptr& pool, + const std::shared_ptr& message_factory, + const std::vector& skip_tests) { + // Check if the entire suite should be skipped (prefix match) + for (const auto& skip : skip_tests) { + if (suite_name == skip || + absl::StartsWith(suite_name, absl::StrCat(skip, "/"))) { + std::cout << "[ SKIPPED SUITE ] " << suite_name << std::endl; + return absl::OkStatus(); + } + } + + std::filesystem::path policy_path = dir_path / "policy.yaml"; + std::filesystem::path tests_path = dir_path / "tests.yaml"; + bool is_yaml = true; + if (!std::filesystem::exists(tests_path)) { + tests_path = dir_path / "tests.textproto"; + is_yaml = false; + } + std::filesystem::path config_path = dir_path / "config.yaml"; + + if (!std::filesystem::exists(policy_path) || + !std::filesystem::exists(tests_path)) { + // Not a valid test suite, assume it's a directory we don't care about. + return absl::OkStatus(); + } + + // Parse Environment Config + cel::Config config; + if (std::filesystem::exists(config_path)) { + std::string config_content; + CEL_RETURN_IF_ERROR( + cel::internal::GetFileContents(config_path.string(), &config_content)); + CEL_ASSIGN_OR_RETURN(config, cel::EnvConfigFromYaml(config_content)); + } + + // Enable default extensions (optional, bindings) in the config + CEL_RETURN_IF_ERROR(config.AddExtensionConfig( + "optional", cel::Config::ExtensionConfig::kLatest)); + CEL_RETURN_IF_ERROR(config.AddExtensionConfig( + "bindings", cel::Config::ExtensionConfig::kLatest)); + + // Set up compiler & runtime environments + cel::Env env; + env.SetDescriptorPool(pool); + cel::RegisterStandardExtensions(env); + env.SetConfig(config); + + cel::EnvRuntime env_runtime; + env_runtime.SetDescriptorPool(pool); + cel::RegisterStandardExtensions(env_runtime); + env_runtime.SetConfig(config); + env_runtime.mutable_runtime_options().enable_qualified_type_identifiers = + true; + + CEL_ASSIGN_OR_RETURN(auto compiler_builder, env.NewCompilerBuilder()); + compiler_builder->GetParserBuilder().GetOptions().enable_optional_syntax = + true; + + CEL_ASSIGN_OR_RETURN(auto compiler, compiler_builder->Build()); + + CEL_ASSIGN_OR_RETURN(auto runtime_builder, + env_runtime.CreateRuntimeBuilder()); + + // Register conformance enums + for (const auto& enum_name : + {"cel.expr.conformance.proto2.GlobalEnum", + "cel.expr.conformance.proto3.GlobalEnum", + "cel.expr.conformance.proto2.TestAllTypes.NestedEnum", + "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"}) { + auto* enum_desc = pool->FindEnumTypeByName(enum_name); + if (enum_desc != nullptr) { + CEL_RETURN_IF_ERROR(cel::extensions::RegisterProtobufEnum( + runtime_builder.type_registry(), enum_desc)); + } + } + + // Register locationCode in runtime + CEL_RETURN_IF_ERROR( + (cel::UnaryFunctionAdapter:: + RegisterGlobalOverload("locationCode", LocationCode, + runtime_builder.function_registry()))); + + CEL_ASSIGN_OR_RETURN(auto runtime, std::move(runtime_builder).Build()); + + // Parse Policy + std::string policy_content; + CEL_RETURN_IF_ERROR( + cel::internal::GetFileContents(policy_path.string(), &policy_content)); + CEL_ASSIGN_OR_RETURN(auto source, + cel::NewSource(policy_content, "policy.yaml")); + auto policy_source = std::make_shared(std::move(source)); + CEL_ASSIGN_OR_RETURN(CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source)); + if (!parse_result.IsValid()) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse policy.yaml in ", suite_name, + "\nIssues:\n", parse_result.FormattedIssues())); + } + const CelPolicy* policy = parse_result.GetPolicy(); + + // Compile Policy (unexpected non-ok status represents a bug) + CEL_ASSIGN_OR_RETURN(CelPolicyValidationResult compile_result, + CompilePolicy(*compiler, *policy)); + + std::string tests_content; + CEL_RETURN_IF_ERROR( + cel::internal::GetFileContents(tests_path.string(), &tests_content)); + TestSuite test_suite; + if (is_yaml) { + CEL_ASSIGN_OR_RETURN(test_suite, + cel::test::ParsePolicyTestSuiteYaml(tests_content)); + } else { + if (!google::protobuf::TextFormat::ParseFromString(tests_content, &test_suite)) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse text proto in ", tests_path.string())); + } + } + + auto runner = std::make_shared( + suite_name, std::move(compiler), std::move(runtime), + std::move(policy_source), std::move(compile_result), pool, + message_factory, input_evaluator, config.GetContextType(), + /*expect_compile_fail=*/absl::StrContains(suite_name, "compile_errors")); + + for (const auto& section : test_suite.sections()) { + std::string section_name = section.name(); + for (const auto& test : section.tests()) { + std::string test_name = test.name(); + std::string full_test_name = + absl::StrCat(suite_name, "/", section_name, "/", test_name); + + bool skip = !ShouldRunTest(full_test_name, skip_tests); + + testing::RegisterTest( + suite_name.c_str(), + absl::StrCat(section_name, "/", test_name).c_str(), nullptr, + test_name.c_str(), __FILE__, __LINE__, + [runner, test, full_test_name, skip]() -> CelPolicyTest* { + return new CelPolicyTest(runner, test, full_test_name, skip); + }); + } + } + return absl::OkStatus(); +} + +void RegisterAllTests() { + // cel::google3-end + std::string testdata_example_flag = absl::GetFlag(FLAGS_testdata_example); + std::vector skip_tests = absl::GetFlag(FLAGS_skip_tests); + + std::string abs_testdata_example = + cel::internal::ResolveRunfilesPath(testdata_example_flag); + ABSL_CHECK(!abs_testdata_example.empty()) + << "Could not find testdata directory: " << testdata_example_flag; + + std::shared_ptr pool = + GetSharedTestingDescriptorPool(); + auto message_factory = + std::make_shared(pool.get()); + message_factory->SetDelegateToGeneratedFactory(true); + auto evaluator_or = InputEvaluator::Create(pool); + ABSL_CHECK_OK(evaluator_or.status()) << "Failed to create input evaluator"; + std::shared_ptr evaluator = std::move(evaluator_or.value()); + + std::filesystem::path testdata_path(abs_testdata_example); + ABSL_CHECK(std::filesystem::exists(testdata_path)) + << "Testdata path does not exist: " << testdata_path; + // walk up to find 'testdata' parent. A work around to portably + // get the expected directory from bazel. + while (!absl::EndsWith(testdata_path.string(), "testdata")) { + testdata_path = testdata_path.parent_path(); + ABSL_CHECK(testdata_path.string().size() > sizeof("testdata")) + << "could not resolve testdata directory"; + } + + for (const auto& entry : + std::filesystem::recursive_directory_iterator(testdata_path)) { + if (!entry.is_directory()) { + continue; + } + std::filesystem::path dir_path = entry.path(); + // Check if this directory has policy.yaml and tests.yaml (or + // tests.textproto) + if (std::filesystem::exists(dir_path / "policy.yaml") && + (std::filesystem::exists(dir_path / "tests.yaml") || + std::filesystem::exists(dir_path / "tests.textproto"))) { + std::string suite_name = absl::StrReplaceAll( + std::filesystem::relative(dir_path, testdata_path).string(), + {{"\\", "/"}}); + + ABSL_CHECK_OK(RegisterTestSuite(dir_path, suite_name, evaluator, pool, + message_factory, skip_tests)); + } + } +} + +} // namespace +} // namespace cel + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + + cel::RegisterAllTests(); + return RUN_ALL_TESTS(); +} diff --git a/internal/BUILD b/internal/BUILD index 0ac5c4e46..6d0efab72 100644 --- a/internal/BUILD +++ b/internal/BUILD @@ -92,6 +92,7 @@ cc_library( hdrs = ["runfiles.h"], deps = [ "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@rules_cc//cc/runfiles", ], diff --git a/internal/runfiles.cc b/internal/runfiles.cc index 259e2e7ca..bffbfa9d1 100644 --- a/internal/runfiles.cc +++ b/internal/runfiles.cc @@ -14,11 +14,14 @@ #include "internal/runfiles.h" +#include +#include #include #include "rules_cc/cc/runfiles/runfiles.h" - #include "absl/log/absl_check.h" + +#include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" @@ -37,4 +40,14 @@ std::string ResolveRunfilesPath(absl::string_view path) { return runfiles->Rlocation(std::string(path)); } +absl::Status GetFileContents(absl::string_view path, std::string* out) { + std::ifstream file{std::string(path)}; + if (!file.is_open()) { + return absl::NotFoundError(absl::StrCat("Failed to open file: ", path)); + } + out->append((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + return absl::OkStatus(); +} + } // namespace cel::internal diff --git a/internal/runfiles.h b/internal/runfiles.h index 643c677b4..11fdcf337 100644 --- a/internal/runfiles.h +++ b/internal/runfiles.h @@ -11,12 +11,15 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// +// Utilities for working with bazel runfiles. #ifndef THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ #define THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ #include +#include "absl/status/status.h" #include "absl/strings/string_view.h" namespace cel::internal { @@ -25,6 +28,9 @@ namespace cel::internal { // Intended for resolving test cases from cel-spec and cel-policy. std::string ResolveRunfilesPath(absl::string_view path); +// Read contents of a file at a resolved path to a string. +absl::Status GetFileContents(absl::string_view path, std::string* out); + } // namespace cel::internal #endif // THIRD_PARTY_CEL_CPP_INTERNAL_RUNFILES_H_ diff --git a/policy/BUILD b/policy/BUILD new file mode 100644 index 000000000..19195be2b --- /dev/null +++ b/policy/BUILD @@ -0,0 +1,239 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "cel_policy", + srcs = [ + "cel_policy.cc", + ], + hdrs = [ + "cel_policy.h", + ], + deps = [ + "//common:source", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + ], +) + +cc_test( + name = "cel_policy_test", + srcs = ["cel_policy_test.cc"], + deps = [ + ":cel_policy", + "//common:source", + "//internal:testing", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "cel_policy_parser", + srcs = [ + "cel_policy_parse_context.cc", + "cel_policy_parse_result.cc", + ], + hdrs = [ + "cel_policy_parse_context.h", + "cel_policy_parse_result.h", + "cel_policy_parser.h", + ], + deps = [ + ":cel_policy", + "//common:source", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "yaml_policy_parser", + srcs = [ + "yaml_policy_parser.cc", + ], + hdrs = ["yaml_policy_parser.h"], + copts = ["-fexceptions"], + features = ["-use_header_modules"], + deps = [ + ":cel_policy", + ":cel_policy_parser", + "//common:source", + "//internal:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@yaml-cpp", + ], +) + +cc_library( + name = "cel_policy_validation_result", + srcs = [ + "cel_policy_validation_result.cc", + ], + hdrs = [ + "cel_policy_validation_result.h", + ], + deps = [ + ":cel_policy", + ":cel_policy_parser", + "//common:ast", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "compiler", + srcs = ["compiler.cc"], + hdrs = ["compiler.h"], + deps = [ + ":cel_policy", + ":cel_policy_parser", + ":cel_policy_validation_result", + "//checker:type_check_issue", + "//checker:validation_result", + "//common:ast", + "//common:ast_rewrite", + "//common:constant", + "//common:container", + "//common:decl", + "//common:expr", + "//common:format_type_name", + "//common:navigable_ast", + "//common:source", + "//common:type", + "//common:type_kind", + "//compiler", + "//internal:status_macros", + "//policy/internal:issue_reporter", + "//policy/internal:optimizer_expr_factory", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "yaml_policy_parser_test", + srcs = [ + "test_custom_yaml_policy_parser.cc", + "yaml_policy_parser_test.cc", + ], + data = [ + "//policy/testdata:policy_testdata", + ], + deps = [ + ":cel_policy", + ":cel_policy_parser", + ":yaml_policy_parser", + "//common:source", + "//internal:runfiles", + "//internal:status_macros", + "//internal:testing", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@yaml-cpp", + ], +) + +cc_test( + name = "compiler_test", + srcs = ["compiler_test.cc"], + data = [ + "//policy/testdata:policy_testdata", + ], + deps = [ + ":cel_policy", + ":cel_policy_parser", + ":cel_policy_validation_result", + ":compiler", + ":yaml_policy_parser", + "//common:ast", + "//common:decl", + "//common:navigable_ast", + "//common:source", + "//common:type", + "//common:value", + "//common:value_testing", + "//compiler", + "//compiler:compiler_factory", + "//compiler:optional", + "//compiler:standard_library", + "//extensions:bindings_ext", + "//internal:runfiles", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//runtime", + "//runtime:activation", + "//runtime:optional_types", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + ], +) + +cc_library( + name = "test_util", + testonly = True, + srcs = ["test_util.cc"], + hdrs = ["test_util.h"], + copts = ["-fexceptions"], + features = ["-use_header_modules"], + deps = [ + "//internal:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_cel_spec//proto/cel/expr:value_cc_proto", + "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", + "@com_google_protobuf//:struct_cc_proto", + "@yaml-cpp", + ], +) diff --git a/policy/cel_policy.cc b/policy/cel_policy.cc new file mode 100644 index 000000000..c2d97edeb --- /dev/null +++ b/policy/cel_policy.cc @@ -0,0 +1,273 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "common/source.h" + +namespace cel { + +namespace { + +std::string IdDebugString(CelPolicyElementId id) { + if (id == -1) { + return ""; + } + return absl::StrCat("#", id, "> "); +} + +std::string IndentBlock(absl::string_view text) { + if (text.empty()) { + return ""; + } + std::vector lines; + for (absl::string_view line : absl::StrSplit(text, '\n')) { + if (line.empty()) { + lines.push_back(""); + } else { + lines.push_back(absl::StrCat(" ", line)); + } + } + return absl::StrJoin(lines, "\n"); +} + +} // namespace + +void CelPolicySource::NoteSourcePosition(CelPolicyElementId id, + SourcePosition position) { + source_positions_[id] = position; +} + +std::optional CelPolicySource::GetSourcePosition( + CelPolicyElementId id) const { + auto it = source_positions_.find(id); + if (it == source_positions_.end()) { + return std::nullopt; + } + return it->second; +} + +std::optional CelPolicySource::GetSourceLocation( + CelPolicyElementId id) const { + auto it = source_positions_.find(id); + if (it == source_positions_.end()) { + return std::nullopt; + } + return policy_source_->GetLocation(it->second); +} + +std::string CelPolicySource::DebugString() const { + std::string result; + + // Sort the source elements in descending order of position + std::vector> sorted_positions; + for (const auto& pair : source_positions_) { + sorted_positions.push_back(pair); + } + std::sort(sorted_positions.begin(), sorted_positions.end(), + [](const auto& a, const auto& b) { + if (a.second == b.second) { + return a.first < b.first; + } + return a.second > b.second; + }); + + result = policy_source_->content().ToString(); + for (const auto& [id, position] : sorted_positions) { + result.insert(position, IdDebugString(id)); + } + return result; +} + +std::string ValueString::DebugString() const { + return absl::StrCat(IdDebugString(id_), "\"", value_, "\""); +} + +std::string Import::DebugString() const { + std::string result; + absl::StrAppend(&result, IdDebugString(id_), "name: ", name_.DebugString()); + return result; +} + +std::string OutputBlock::DebugString() const { + std::string result; + absl::StrAppend(&result, "output: ", output_.DebugString()); + if (explanation_.has_value()) { + absl::StrAppend(&result, "\nexplanation: ", explanation_->DebugString()); + } + return result; +} + +Match::Match(const Match& other) + : id_(other.id_), condition_(other.condition_) { + if (std::holds_alternative(other.result_)) { + result_ = std::get(other.result_); + } else if (std::holds_alternative(other.result_)) { + result_ = std::get(other.result_); + } else { + result_ = + std::make_unique(*std::get>(other.result_)); + } +} + +Match& Match::operator=(const Match& other) { + if (this != &other) { + id_ = other.id_; + condition_ = other.condition_; + if (std::holds_alternative(other.result_)) { + result_ = std::get(other.result_); + } else if (std::holds_alternative(other.result_)) { + result_ = std::get(other.result_); + } else { + result_ = std::make_unique( + *std::get>(other.result_)); + } + } + return *this; +} + +std::string Match::DebugString() const { + std::string result; + absl::StrAppend(&result, IdDebugString(id_), "match: {\n"); + if (condition_.has_value()) { + absl::StrAppend(&result, " condition: ", condition_->DebugString(), "\n"); + } + if (has_rule()) { + absl::StrAppend(&result, " result:\n", + IndentBlock(IndentBlock(rule().DebugString())), "\n"); + } else { + absl::StrAppend(&result, " result: {\n", + IndentBlock(IndentBlock(output_block().DebugString())), + "\n }\n"); + } + absl::StrAppend(&result, "}"); + return result; +} + +std::string Variable::DebugString() const { + std::string result; + absl::StrAppend(&result, "variable: {\n"); + absl::StrAppend(&result, " name: ", name_.DebugString(), "\n"); + absl::StrAppend(&result, " expression: ", expression_.DebugString(), "\n"); + if (description_.has_value()) { + absl::StrAppend(&result, " description: ", description_->DebugString(), + "\n"); + } + if (display_name_.has_value()) { + absl::StrAppend(&result, " display_name: ", display_name_->DebugString(), + "\n"); + } + absl::StrAppend(&result, "}"); + return result; +} + +std::string Rule::DebugString() const { + std::string result; + absl::StrAppend(&result, IdDebugString(id_), "rule: {\n"); + if (rule_id_.has_value()) { + absl::StrAppend(&result, " rule_id: ", rule_id_->DebugString(), "\n"); + } + if (description_.has_value()) { + absl::StrAppend(&result, " description: ", description_->DebugString(), + "\n"); + } + for (const Variable& variable : variables_) { + absl::StrAppend(&result, IndentBlock(variable.DebugString()), "\n"); + } + for (const Match& match : matches_) { + absl::StrAppend(&result, IndentBlock(match.DebugString()), "\n"); + } + absl::StrAppend(&result, "}"); + return result; +} + +std::string MetadataValueDebugString(std::any value) { + if (value.type() == typeid(std::monostate)) { + return "null"; + } + if (value.type() == typeid(ValueString)) { + return std::any_cast(value).DebugString(); + } + if (value.type() == typeid(bool)) { + return std::any_cast(value) ? "true" : "false"; + } + if (value.type() == typeid(int)) { + return absl::StrCat(std::any_cast(value)); + } + if (value.type() == typeid(std::string)) { + return std::any_cast(value); + } + return absl::StrCat("typeid: ", value.type().name()); +} + +std::string CelPolicy::DebugString() const { + std::string result; + absl::StrAppend(&result, "CelPolicy{\n"); + absl::StrAppend( + &result, + " ===========================================================\n"); + absl::StrAppend(&result, IndentBlock(IndentBlock(source_->DebugString())), + "\n"); + absl::StrAppend( + &result, + " ===========================================================\n"); + absl::StrAppend(&result, " name: ", name_.DebugString(), "\n"); + if (description_.has_value()) { + absl::StrAppend(&result, " description: ", description_->DebugString(), + "\n"); + } + if (display_name_.has_value()) { + absl::StrAppend(&result, " display_name: ", display_name_->DebugString(), + "\n"); + } + if (!metadata_.empty()) { + std::vector sorted_keys; + for (const auto& [key, _] : metadata_) { + sorted_keys.push_back(key); + } + std::sort(sorted_keys.begin(), sorted_keys.end()); + + absl::StrAppend(&result, " metadata: {\n"); + for (const auto& key : sorted_keys) { + const auto& value = metadata_.at(key); + absl::StrAppend(&result, " ", key, ": ", + MetadataValueDebugString(value), "\n"); + } + absl::StrAppend(&result, " }\n"); + } + if (!imports_.empty()) { + absl::StrAppend(&result, " imports:\n"); + for (const Import& import : imports_) { + absl::StrAppend(&result, " ", import.DebugString(), "\n"); + } + } + absl::StrAppend(&result, IndentBlock(rule_.DebugString()), "\n"); + absl::StrAppend(&result, "}"); + return result; +} + +} // namespace cel diff --git a/policy/cel_policy.h b/policy/cel_policy.h new file mode 100644 index 000000000..af8f7c977 --- /dev/null +++ b/policy/cel_policy.h @@ -0,0 +1,320 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "common/source.h" + +namespace cel { + +using CelPolicyElementId = int32_t; + +class CelPolicySource { + public: + explicit CelPolicySource(cel::SourcePtr policy_source) + : policy_source_(std::move(policy_source)) {} + + const Source* absl_nonnull content() const { return policy_source_.get(); } + + void NoteSourcePosition(CelPolicyElementId id, SourcePosition position); + + std::optional GetSourcePosition(CelPolicyElementId id) const; + + std::optional GetSourceLocation(CelPolicyElementId id) const; + + std::string DebugString() const; + + private: + cel::SourcePtr policy_source_; + absl::flat_hash_map source_positions_; +}; + +class ValueString { + public: + ValueString() : id_(-1) {} + + explicit ValueString(CelPolicyElementId id, absl::string_view value) + : id_(id), value_(value) {} + + CelPolicyElementId id() const { return id_; } + absl::string_view value() const { return value_; } + + std::string DebugString() const; + + private: + CelPolicyElementId id_; + std::string value_; +}; + +class Import { + public: + Import(CelPolicyElementId id, ValueString name) + : id_(id), name_(std::move(name)) {} + CelPolicyElementId id() const { return id_; } + const ValueString& name() const { return name_; } + + std::string DebugString() const; + + private: + CelPolicyElementId id_; + ValueString name_; +}; + +// Defines a variable that can be used in CEL expressions within the policy. +// Variables are evaluated once and stored in the activation context. +class Variable { + public: + const ValueString& name() const { return name_; } + void set_name(ValueString name) { name_ = std::move(name); } + + const ValueString& expression() const { return expression_; } + void set_expression(ValueString expression) { + expression_ = std::move(expression); + } + + std::optional description() const { return description_; } + void set_description(ValueString description) { + description_ = std::move(description); + } + + std::optional display_name() const { return display_name_; } + void set_display_name(ValueString display_name) { + display_name_ = std::move(display_name); + } + + std::string DebugString() const; + + private: + ValueString name_; + ValueString expression_; + std::optional description_; + std::optional display_name_; +}; + +class Rule; + +class OutputBlock { + public: + OutputBlock() = default; + OutputBlock(ValueString output, std::optional explanation) + : output_(std::move(output)), explanation_(std::move(explanation)) {} + + const ValueString& output() const { return output_; } + void set_output(ValueString output) { output_ = std::move(output); } + + const std::optional& explanation() const { return explanation_; } + void set_explanation(ValueString explanation) { + explanation_ = std::move(explanation); + } + + std::string DebugString() const; + + private: + ValueString output_; + std::optional explanation_; +}; + +// Defines a match condition and result. +// If the result is a Rule, it is considered a sub-rule and will be evaluated +// only if the match condition evaluates to true. +class Match { + public: + Match() = default; + Match(const Match& other); + Match& operator=(const Match& other); + + CelPolicyElementId id() const; + void set_id(CelPolicyElementId id); + + bool has_condition() const; + std::optional condition() const; + void set_condition(ValueString condition); + + bool has_output_block() const; + const OutputBlock& output_block() const; + OutputBlock& mutable_output_block(); + + bool has_rule() const; + const Rule& rule() const; + Rule& mutable_rule(); + + void set_result(OutputBlock result); + void set_result(std::unique_ptr result); + + std::string DebugString() const; + + private: + CelPolicyElementId id_ = -1; + std::optional condition_; + std::variant> result_; +}; + +// Rule is the body of the policy and contains a list of variables and matches. +// Variables are evaluated once and stored in the activation context. +// Matches are evaluated in order and the first match is returned. If the +// match contains a sub-rule, the sub-rule is evaluated only if the match +// condition evaluates to true. +class Rule { + public: + Rule() = default; + Rule(const Rule& other) = default; + + CelPolicyElementId id() const { return id_; } + void set_id(CelPolicyElementId id) { id_ = id; } + + const std::optional& rule_id() const { return rule_id_; } + void set_rule_id(ValueString rule_id) { rule_id_ = std::move(rule_id); } + + const std::optional& description() const { return description_; } + void set_description(ValueString description) { + description_ = std::move(description); + } + + const std::vector& variables() const { return variables_; } + std::vector& mutable_variables() { return variables_; } + + const std::vector& matches() const { return matches_; } + std::vector& mutable_matches() { return matches_; } + + std::string DebugString() const; + + private: + CelPolicyElementId id_ = -1; + std::optional rule_id_; + std::optional description_; + std::vector variables_; + std::vector matches_; +}; + +// CelPolicy is the top-level policy object. +// It contains a source, name, description, display name, imports, and a rule. +// The source is the CEL policy source code. +// The name, description, and display name are metadata about the policy. +// The rule is the main body of the policy. +class CelPolicy { + public: + explicit CelPolicy(std::shared_ptr source) + : source_(std::move(source)) {} + + CelPolicy(const CelPolicy& other) = default; + CelPolicy& operator=(const CelPolicy& other) = default; + + const CelPolicySource* absl_nullable source() const { return source_.get(); } + const std::shared_ptr& source_ptr() const { return source_; } + + const ValueString& name() const { return name_; } + void set_name(ValueString name) { name_ = std::move(name); } + + std::optional description() const { return description_; } + void set_description(ValueString description) { + description_ = std::move(description); + } + std::optional display_name() const { return display_name_; } + void set_display_name(ValueString display_name) { + display_name_ = std::move(display_name); + } + const absl::flat_hash_map& metadata() const { + return metadata_; + } + absl::flat_hash_map& mutable_metadata() { + return metadata_; + } + const std::vector& imports() const { return imports_; } + std::vector& mutable_imports() { return imports_; } + + const Rule& rule() const { return rule_; } + Rule& mutable_rule() { return rule_; } + + std::string DebugString() const; + + private: + std::shared_ptr source_; + ValueString name_; + std::optional description_; + std::optional display_name_; + absl::flat_hash_map metadata_; + std::vector imports_; + Rule rule_; +}; + +// Implementation details. + +inline CelPolicyElementId Match::id() const { return id_; } +inline void Match::set_id(CelPolicyElementId id) { id_ = id; } + +inline bool Match::has_condition() const { return condition_.has_value(); } + +inline std::optional Match::condition() const { + return condition_; +} + +inline void Match::set_condition(ValueString condition) { + condition_ = std::move(condition); +} + +inline bool Match::has_output_block() const { + return std::holds_alternative(result_); +} + +inline const OutputBlock& Match::output_block() const { + ABSL_DCHECK(std::holds_alternative(result_)); + return std::get(result_); +} + +inline OutputBlock& Match::mutable_output_block() { + if (!std::holds_alternative(result_)) { + result_ = OutputBlock(); + } + return std::get(result_); +} + +inline bool Match::has_rule() const { + return std::holds_alternative>(result_); +} + +inline const Rule& Match::rule() const { + ABSL_DCHECK(std::holds_alternative>(result_)); + return *std::get>(result_); +} + +inline Rule& Match::mutable_rule() { + ABSL_DCHECK(std::holds_alternative>(result_)); + return *std::get>(result_); +} + +inline void Match::set_result(OutputBlock result) { + result_ = std::move(result); +} + +inline void Match::set_result(std::unique_ptr result) { + result_ = std::move(result); +} + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_H_ diff --git a/policy/cel_policy_parse_context.cc b/policy/cel_policy_parse_context.cc new file mode 100644 index 000000000..66861d085 --- /dev/null +++ b/policy/cel_policy_parse_context.cc @@ -0,0 +1,49 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy_parse_context.h" + +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel { + +CelPolicy& CelPolicyParseContext::policy() const { + ABSL_CHECK(policy_ != nullptr) + << "CelPolicyParseContext::policy() called after GetResult()"; + return *policy_; +} + +CelPolicyParseResult CelPolicyParseContext::GetResult() { + if (policy_ != nullptr && issues_.empty()) { + return CelPolicyParseResult(std::move(policy_source_), std::move(policy_), + std::move(issues_)); + } + policy_.reset(); + return CelPolicyParseResult(std::move(policy_source_), nullptr, + std::move(issues_)); +} + +void CelPolicyParseContext::ReportError(CelPolicyElementId element_id, + std::string_view message) { + issues_.push_back(CelPolicyIssue(element_id, std::string(message))); +} + +} // namespace cel diff --git a/policy/cel_policy_parse_context.h b/policy/cel_policy_parse_context.h new file mode 100644 index 000000000..6482fa1ae --- /dev/null +++ b/policy/cel_policy_parse_context.h @@ -0,0 +1,65 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_CONTEXT_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_CONTEXT_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel { + +// A mutable context for parsing a CelPolicy. An instance of this class is +// created for each policy parse and is passed to the parser, which is meant to +// be stateless. +// +// Parsers call methods on this class to report issues and populate the policy +// being parsed. Call GetResult() to obtain the resulting CelPolicyParseResult, +// which takes ownership of the parsed policy. Do not use the context after +// calling GetResult(). +class CelPolicyParseContext { + public: + explicit CelPolicyParseContext(std::shared_ptr policy_source) + : policy_source_(std::move(policy_source)), + policy_(std::make_unique(policy_source_)) {} + + CelPolicySource& policy_source() const { return *policy_source_; } + + // Returns the policy being parsed. It should not be used after + // calling GetResult(). + CelPolicy& policy() const; + + // The context should not be used after calling GetResult(). + CelPolicyParseResult GetResult(); + + // Reports an error for the given element with the given error message. + void ReportError(CelPolicyElementId id, std::string_view message); + + CelPolicyElementId next_element_id() { return next_element_id_++; } + + private: + std::shared_ptr policy_source_; + CelPolicyElementId next_element_id_ = 0; + std::vector issues_; + std::unique_ptr policy_; +}; + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_CONTEXT_H_ diff --git a/policy/cel_policy_parse_result.cc b/policy/cel_policy_parse_result.cc new file mode 100644 index 000000000..32d6431bb --- /dev/null +++ b/policy/cel_policy_parse_result.cc @@ -0,0 +1,91 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy_parse_result.h" + +#include +#include + +#include "absl/base/nullability.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "common/source.h" +#include "policy/cel_policy.h" + +namespace cel { +namespace { + +absl::string_view SeverityString(CelPolicyIssue::Severity severity) { + switch (severity) { + case CelPolicyIssue::Severity::kInformation: + return "INFORMATION"; + case CelPolicyIssue::Severity::kWarning: + return "WARNING"; + case CelPolicyIssue::Severity::kError: + return "ERROR"; + case CelPolicyIssue::Severity::kDeprecated: + return "DEPRECATED"; + default: + return "SEVERITY_UNSPECIFIED"; + } +} + +} // namespace + +std::string CelPolicyIssue::ToDisplayString( + const CelPolicySource* absl_nullable source) const { + SourceLocation location; + std::string description; + std::string snippet; + if (source != nullptr) { + if (relative_position_) { + std::optional base = + source->GetSourcePosition(element_id_); + if (element_id_ == -1) { + base.emplace(0); + } + if (base) { + location = source->content() + ->GetLocation(*base + *relative_position_) + .value_or(SourceLocation{}); + } + } else { + location = + source->GetSourceLocation(element_id_).value_or(SourceLocation{}); + } + description = std::string(source->content()->description()); + snippet = source->content()->DisplayErrorLocation(location); + } + + const int display_column = location.column >= 0 ? location.column + 1 : -1; + + return absl::StrFormat("%s: %s:%d:%d: %s%s", SeverityString(severity_), + description, location.line, display_column, message_, + snippet); +} + +std::string CelPolicyParseResult::FormattedIssues() const { + std::string formatted_issues; + for (const CelPolicyIssue& issue : issues_) { + if (!formatted_issues.empty()) { + absl::StrAppend(&formatted_issues, "\n"); + } + absl::StrAppend(&formatted_issues, issue.ToDisplayString(*policy_source_)); + } + return formatted_issues; +} + +} // namespace cel diff --git a/policy/cel_policy_parse_result.h b/policy/cel_policy_parse_result.h new file mode 100644 index 000000000..2bf80b1ce --- /dev/null +++ b/policy/cel_policy_parse_result.h @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_RESULT_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_RESULT_H_ + +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/source.h" +#include "policy/cel_policy.h" + +namespace cel { + +class CelPolicyIssue { + public: + enum class Severity { kInformation, kDeprecated, kWarning, kError }; + + CelPolicyIssue(CelPolicyElementId element_id, absl::string_view message) + : element_id_(element_id), message_(message) {} + CelPolicyIssue(CelPolicyElementId element_id, Severity severity, + absl::string_view message) + : element_id_(element_id), severity_(severity), message_(message) {} + CelPolicyIssue(CelPolicyElementId element_id, + SourcePosition relative_position, absl::string_view message) + : element_id_(element_id), + relative_position_(relative_position), + message_(message) {} + CelPolicyIssue(CelPolicyElementId element_id, + SourcePosition relative_position, Severity severity, + absl::string_view message) + : element_id_(element_id), + relative_position_(relative_position), + severity_(severity), + message_(message) {} + + std::string ToDisplayString( + const CelPolicySource* absl_nullable source) const; + std::string ToDisplayString(const CelPolicySource& source) const { + return ToDisplayString(&source); + } + + Severity severity() const { return severity_; } + absl::string_view message() const { return message_; } + + private: + CelPolicyElementId element_id_; + std::optional relative_position_; + Severity severity_ = Severity::kError; + std::string message_; +}; + +class CelPolicyParseResult { + public: + explicit CelPolicyParseResult(std::shared_ptr policy_source, + std::unique_ptr policy, + std::vector issues) + : policy_source_(std::move(policy_source)), + policy_(std::move(policy)), + issues_(std::move(issues)) {} + + bool IsValid() const { return policy_ != nullptr; } + + const CelPolicy* absl_nullable GetPolicy() const { return policy_.get(); } + + absl::StatusOr> ReleasePolicy() { + if (policy_ == nullptr) { + return absl::FailedPreconditionError( + "CelPolicyParseResult is empty. Check for Issues."); + } + return std::move(policy_); + } + + absl::Span GetIssues() const { return issues_; } + + std::string FormattedIssues() const; + + private: + std::shared_ptr policy_source_; + absl_nullable std::unique_ptr policy_; + std::vector issues_; +}; + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSE_RESULT_H_ diff --git a/policy/cel_policy_parser.h b/policy/cel_policy_parser.h new file mode 100644 index 000000000..0a11c9e68 --- /dev/null +++ b/policy/cel_policy_parser.h @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSER_H_ + +#include "absl/status/status.h" +#include "policy/cel_policy_parse_context.h" + +namespace cel { + +// A policy parser for a given policy format. The type `T` parameter is the +// representation of the input file format, such as `` for YAML. +// +// Parsers are intended to be stateless: all state, including the resulting +// policy and any issues encountered, should be kept in the context passed to +// the `ParsePolicy` method. +template +class CelPolicyParser { + public: + virtual ~CelPolicyParser() = default; + + // Parses the input and populates a CelPolicy in the context. + virtual absl::Status ParsePolicy(CelPolicyParseContext& ctx) const = 0; +}; + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_PARSER_H_ diff --git a/policy/cel_policy_test.cc b/policy/cel_policy_test.cc new file mode 100644 index 000000000..640247e7f --- /dev/null +++ b/policy/cel_policy_test.cc @@ -0,0 +1,220 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy.h" + +#include +#include +#include +#include + +#include "absl/strings/str_replace.h" +#include "common/source.h" +#include "internal/testing.h" + +namespace cel { +namespace { + +using testing::Field; +using testing::Optional; +using testing::SizeIs; + +TEST(CelPolicyBuilderTest, Build) { + CelPolicyElementId next_id = 1; + ASSERT_OK_AND_ASSIGN(SourcePtr source, NewSource("CEL\n policy\n source")); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + CelPolicy policy(policy_source); + policy.set_name(ValueString(next_id++, "test_policy")); + policy.set_description(ValueString(next_id++, "test_description")); + policy.set_display_name(ValueString(next_id++, "test_display_name")); + ValueString import1_name = ValueString(next_id++, "test_import1"); + policy.mutable_imports().push_back(Import(next_id++, import1_name)); + ValueString import2_name = ValueString(next_id++, "test_import2"); + policy.mutable_imports().push_back(Import(next_id++, import2_name)); + + Rule& rule = policy.mutable_rule(); + rule.set_id(next_id++); + rule.set_rule_id(ValueString(next_id++, "test_rule_id")); + rule.set_description(ValueString(next_id++, "test_rule_description")); + + Variable variable; + variable.set_name(ValueString(next_id++, "test_variable")); + variable.set_expression(ValueString(next_id++, "test_expression")); + variable.set_description(ValueString(next_id++, "test_variable_description")); + variable.set_display_name( + ValueString(next_id++, "test_variable_display_name")); + + Match match1; + match1.set_id(next_id++); + match1.set_condition(ValueString(next_id++, "test_condition")); + CelPolicyElementId output_id = next_id++; + CelPolicyElementId explanation_id = next_id++; + match1.set_result( + OutputBlock(ValueString(output_id, "test_result"), + ValueString(explanation_id, "test_explanation"))); + + Match match2; + match2.set_id(next_id++); + match2.set_condition(ValueString(next_id++, "test_condition2")); + + auto sub_rule = std::make_unique(); + sub_rule->set_id(next_id++); + sub_rule->set_rule_id(ValueString(next_id++, "sub_rule_id")); + sub_rule->set_description(ValueString(next_id++, "sub_rule_description")); + Match sub_rule_match; + sub_rule_match.set_id(next_id++); + sub_rule_match.set_condition(ValueString(next_id++, "sub_rule_condition")); + sub_rule_match.set_result( + OutputBlock(ValueString(next_id++, "sub_rule_result"), std::nullopt)); + sub_rule->mutable_matches().push_back(sub_rule_match); + + match2.set_result(std::move(sub_rule)); + + rule.mutable_variables().push_back(variable); + rule.mutable_matches().push_back(match1); + rule.mutable_matches().push_back(match2); + + EXPECT_EQ(policy.name().value(), "test_policy"); + ASSERT_TRUE(policy.description().has_value()); + EXPECT_EQ(policy.description()->value(), "test_description"); + ASSERT_TRUE(policy.display_name().has_value()); + EXPECT_EQ(policy.display_name()->value(), "test_display_name"); + + ASSERT_THAT(policy.imports(), SizeIs(2)); + + EXPECT_EQ(policy.imports()[0].name().value(), "test_import1"); + EXPECT_EQ(policy.imports()[1].name().value(), "test_import2"); + ASSERT_TRUE(policy.rule().rule_id().has_value()); + EXPECT_EQ(policy.rule().rule_id()->value(), "test_rule_id"); + ASSERT_TRUE(policy.rule().description().has_value()); + EXPECT_EQ(policy.rule().description()->value(), "test_rule_description"); + + ASSERT_THAT(policy.rule().variables(), SizeIs(1)); + + EXPECT_EQ(policy.rule().variables()[0].name().value(), "test_variable"); + EXPECT_EQ(policy.rule().variables()[0].expression().value(), + "test_expression"); + ASSERT_TRUE(policy.rule().variables()[0].description().has_value()); + EXPECT_EQ(policy.rule().variables()[0].description()->value(), + "test_variable_description"); + ASSERT_TRUE(policy.rule().variables()[0].display_name().has_value()); + EXPECT_EQ(policy.rule().variables()[0].display_name()->value(), + "test_variable_display_name"); + + ASSERT_THAT(policy.rule().matches(), SizeIs(2)); + + EXPECT_EQ(policy.rule().matches()[0].condition().value().value(), + "test_condition"); + ASSERT_TRUE(policy.rule().matches()[0].has_output_block()); + EXPECT_EQ(policy.rule().matches()[0].output_block().output().value(), + "test_result"); + ASSERT_TRUE( + policy.rule().matches()[0].output_block().explanation().has_value()); + EXPECT_EQ(policy.rule().matches()[0].output_block().explanation()->value(), + "test_explanation"); + + EXPECT_EQ(policy.rule().matches()[1].condition().value().value(), + "test_condition2"); + ASSERT_TRUE(policy.rule().matches()[1].has_rule()); + ASSERT_TRUE(policy.rule().matches()[1].rule().rule_id().has_value()); + EXPECT_EQ(policy.rule().matches()[1].rule().rule_id()->value(), + "sub_rule_id"); + ASSERT_TRUE(policy.rule().matches()[1].rule().description().has_value()); + EXPECT_EQ(policy.rule().matches()[1].rule().description()->value(), + "sub_rule_description"); + ASSERT_THAT(policy.rule().matches()[1].rule().matches(), SizeIs(1)); + EXPECT_EQ(policy.rule() + .matches()[1] + .rule() + .matches()[0] + .condition() + .value() + .value(), + "sub_rule_condition"); + + std::string actual = policy.DebugString(); + EXPECT_EQ(actual, absl::StrReplaceAll(R"(CelPolicy{ + =========================================================== + CEL + policy + source + =========================================================== + name: #1> "test_policy" + description: #2> "test_description" + display_name: #3> "test_display_name" + imports: + #5> name: #4> "test_import1" + #7> name: #6> "test_import2" + #8> rule: { + rule_id: #9> "test_rule_id" + description: #10> "test_rule_description" + variable: { + name: #11> "test_variable" + expression: #12> "test_expression" + description: #13> "test_variable_description" + display_name: #14> "test_variable_display_name" + } + #15> match: { + condition: #16> "test_condition" + result: { + output: #17> "test_result" + explanation: #18> "test_explanation" + } + } + #19> match: { + condition: #20> "test_condition2" + result: + #21> rule: { + rule_id: #22> "sub_rule_id" + description: #23> "sub_rule_description" + #24> match: { + condition: #25> "sub_rule_condition" + result: { + output: #26> "sub_rule_result" + } + } + } + } + } + })", + {{"\n ", "\n"}})); +} + +TEST(CelPolicySourceTest, Build) { + std::string source = + "name: test_policy\n imports:\n - name: test_import\n"; + + ASSERT_OK_AND_ASSIGN(SourcePtr source_ptr, NewSource(source)); + CelPolicySource policy_source(std::move(source_ptr)); + policy_source.NoteSourcePosition(1, source.find("test_policy")); + policy_source.NoteSourcePosition(2, source.find("test_import")); + + EXPECT_THAT(policy_source.GetSourcePosition(1), Optional(6)); + EXPECT_THAT(policy_source.GetSourceLocation(1), + Optional(AllOf(Field(&SourceLocation::line, 1), + Field(&SourceLocation::column, 6)))); + EXPECT_THAT(policy_source.GetSourcePosition(2), Optional(44)); + EXPECT_THAT(policy_source.GetSourceLocation(2), + Optional(AllOf(Field(&SourceLocation::line, 3), + Field(&SourceLocation::column, 13)))); + EXPECT_EQ(policy_source.GetSourcePosition(3), std::nullopt); + EXPECT_EQ(policy_source.GetSourceLocation(3), std::nullopt); + EXPECT_EQ( + policy_source.DebugString(), + "name: #1> test_policy\n imports:\n - name: #2> test_import\n"); +} + +} // namespace +} // namespace cel diff --git a/policy/cel_policy_validation_result.cc b/policy/cel_policy_validation_result.cc new file mode 100644 index 000000000..e257f064c --- /dev/null +++ b/policy/cel_policy_validation_result.cc @@ -0,0 +1,32 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/cel_policy_validation_result.h" + +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel { + +std::string CelPolicyValidationResult::FormatIssues() const { + return absl::StrJoin( + issues_, "\n", [this](std::string* out, const CelPolicyIssue& issue) { + absl::StrAppend(out, issue.ToDisplayString(source_.get())); + }); +} + +} // namespace cel diff --git a/policy/cel_policy_validation_result.h b/policy/cel_policy_validation_result.h new file mode 100644 index 000000000..bddb9a3ca --- /dev/null +++ b/policy/cel_policy_validation_result.h @@ -0,0 +1,84 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_VALIDATION_RESULT_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_VALIDATION_RESULT_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel { + +// CelPolicyValidationResult holds the result of policy compilation. +// +// Policy compilation/validation errors are captured in issues. +class CelPolicyValidationResult { + public: + CelPolicyValidationResult( + std::unique_ptr ast, std::vector issues, + std::shared_ptr source = nullptr) + : ast_(std::move(ast)), + issues_(std::move(issues)), + source_(std::move(source)) {} + + explicit CelPolicyValidationResult( + std::vector issues, + std::shared_ptr source = nullptr) + : ast_(nullptr), issues_(std::move(issues)), source_(std::move(source)) {} + + // Returns true if validation succeeded and an AST is present. + bool IsValid() const { return ast_ != nullptr; } + + // Returns the AST if validation was successful. + const Ast* absl_nullable GetAst() const { return ast_.get(); } + + // Moves out and returns the AST. + absl::StatusOr> ReleaseAst() { + if (ast_ == nullptr) { + return absl::FailedPreconditionError( + "CelPolicyValidationResult is empty. Check for CelPolicyIssues."); + } + return std::move(ast_); + } + + // Returns the list of issues encountered during compilation. + absl::Span GetIssues() const { return issues_; } + + // Returns the contained policy source, if any. + const CelPolicySource* absl_nullable GetSource() const { + return source_.get(); + } + + // Returns a formatted error string of the compiled issues. + std::string FormatIssues() const; + + private: + absl_nullable std::unique_ptr ast_; + std::vector issues_; + std::shared_ptr source_; +}; + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_CEL_POLICY_VALIDATION_RESULT_H_ diff --git a/policy/compiler.cc b/policy/compiler.cc new file mode 100644 index 000000000..7a892447c --- /dev/null +++ b/policy/compiler.cc @@ -0,0 +1,1058 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/compiler.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/base/nullability.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "checker/type_check_issue.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/ast_rewrite.h" +#include "common/constant.h" +#include "common/container.h" +#include "common/decl.h" +#include "common/expr.h" +#include "common/format_type_name.h" +#include "common/navigable_ast.h" +#include "common/source.h" +#include "common/type.h" +#include "common/type_kind.h" +#include "compiler/compiler.h" +#include "internal/status_macros.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_validation_result.h" +#include "policy/internal/issue_reporter.h" +#include "policy/internal/optimizer_expr_factory.h" +#include "google/protobuf/arena.h" + +namespace cel { +namespace { + +constexpr absl::string_view kCelBlock = "cel.@block"; + +enum class RuleSemantics { + // TODO(b/506179116): will also need "aggregate" or similar concept. + kFirstMatch, + + kNotForUseWithExhaustiveSwitchStatements, +}; + +template +void AbslStringify(Sink& s, RuleSemantics semantics) { + switch (semantics) { + case RuleSemantics::kFirstMatch: + s.Append("first_match"); + return; + default: + s.Append(""); + return; + } +} + +struct EmbeddedAst { + CelPolicyElementId id; + std::unique_ptr ast; +}; + +struct CompiledVariable { + std::string ident; + EmbeddedAst ast; +}; + +struct CompiledOutputBlock { + EmbeddedAst output_ast; + cel::Type result_type; + std::optional explanation_ast; +}; + +struct CompiledRule; + +struct CompiledMatch { + using Production = + std::variant absl_nonnull, + CompiledOutputBlock>; + + CelPolicyElementId id; + std::optional condition; + Production production; +}; + +struct CompiledRule { + CelPolicyElementId id; + std::vector variables; + std::vector matches; + // Not set if cannot be determined. + std::optional result_type; +}; + +std::optional GetOutputType( + const CompiledMatch::Production& production) { + return std::visit( + [](const auto& production) -> std::optional { + if constexpr (std::is_same_v, + CompiledOutputBlock>) { + return production.result_type; + } else if constexpr (std::is_same_v, + std::unique_ptr>) { + return production->result_type; + } + return std::nullopt; + }, + production); +} + +// Internal representation of the compiled policy elements. +// +// This is used for checking the component expression before composing into the +// final AST based on the provided rule semantics. +class IntermediateCompiledPolicy { + public: + CompiledRule& mutable_root_rule() { return root_rule_; } + + const CompiledRule& root_rule() const { return root_rule_; } + + void set_name(absl::string_view name) { name_ = name; } + absl::string_view name() const { return name_; } + void set_display_name(absl::string_view display_name) { + display_name_ = display_name; + } + absl::string_view display_name() const { return display_name_; } + void set_description(absl::string_view description) { + description_ = description; + } + absl::string_view description() const { return description_; } + + void set_semantics(RuleSemantics semantics) { semantics_ = semantics; } + RuleSemantics semantics() const { return semantics_; } + + private: + std::string name_; + std::string display_name_; + std::string description_; + RuleSemantics semantics_ = RuleSemantics::kFirstMatch; + + CompiledRule root_rule_; +}; + +CelPolicyIssue::Severity MapSeverity(cel::TypeCheckIssue::Severity severity) { + switch (severity) { + case cel::TypeCheckIssue::Severity::kError: + return CelPolicyIssue::Severity::kError; + case cel::TypeCheckIssue::Severity::kWarning: + return CelPolicyIssue::Severity::kWarning; + case cel::TypeCheckIssue::Severity::kDeprecated: + return CelPolicyIssue::Severity::kDeprecated; + default: + return CelPolicyIssue::Severity::kError; + } +} + +bool IsWrapperOf(cel::TypeKind wrapper_kind, cel::TypeKind primitive_kind) { + switch (wrapper_kind) { + case cel::TypeKind::kBoolWrapper: + return primitive_kind == cel::TypeKind::kBool; + case cel::TypeKind::kIntWrapper: + return primitive_kind == cel::TypeKind::kInt; + case cel::TypeKind::kUintWrapper: + return primitive_kind == cel::TypeKind::kUint; + case cel::TypeKind::kDoubleWrapper: + return primitive_kind == cel::TypeKind::kDouble; + case cel::TypeKind::kStringWrapper: + return primitive_kind == cel::TypeKind::kString; + case cel::TypeKind::kBytesWrapper: + return primitive_kind == cel::TypeKind::kBytes; + default: + return false; + } +} + +cel::Type FilterSpecialTypes(cel::Type type) { + if (type.IsTypeParam()) { + // Free type param should not appear in the output type, but if it does, + // force it to dyn. + return DynType(); + } + if (type.IsEnum()) { + return IntType{}; + } + if (type.IsError()) { + return DynType(); + } + if (type.IsType()) { + // drop parameters so all type types are compatible. + return TypeType{}; + } + return type; +} + +// Returns true if `from` is assignable to `to`. +// +// Slightly adjusted from the standard routine to cover some edge cases around +// null and wrappers. +// +// TODO(b/522391716): try to standardize assignability checks. +bool OutputTypeIsAssignable(cel::Type from, cel::Type to) { + from = FilterSpecialTypes(from); + to = FilterSpecialTypes(to); + + // Any and dyn are assignable to/from everything. + if (from.kind() == cel::TypeKind::kAny || + from.kind() == cel::TypeKind::kDyn || to.kind() == cel::TypeKind::kAny || + to.kind() == cel::TypeKind::kDyn) { + return true; + } + + // Wrappers auto-unwrap. + if (IsWrapperOf(from.kind(), to.kind()) || + IsWrapperOf(to.kind(), from.kind())) { + return true; + } + + // Null is assignable to anything that is message-like. + if (from.kind() == cel::TypeKind::kNull) { + switch (to.kind()) { + case cel::TypeKind::kNull: + case cel::TypeKind::kStruct: + case cel::TypeKind::kOpaque: + case cel::TypeKind::kTimestamp: + case cel::TypeKind::kDuration: + case cel::TypeKind::kBytesWrapper: + case cel::TypeKind::kBoolWrapper: + case cel::TypeKind::kIntWrapper: + case cel::TypeKind::kUintWrapper: + case cel::TypeKind::kDoubleWrapper: + case cel::TypeKind::kStringWrapper: + return true; + default: + return false; + } + } + + if (from.kind() != to.kind()) { + return false; + } + + if (from.name() != to.name()) { + return false; + } + + if (from.GetParameters().size() != to.GetParameters().size()) { + return false; + } + + for (int i = 0; i < from.GetParameters().size(); ++i) { + if (!OutputTypeIsAssignable(from.GetParameters()[i], + to.GetParameters()[i])) { + return false; + } + } + + return true; +} + +bool OutputTypeIsCompatible(cel::Type from, cel::Type to) { + // We don't handle widening like in a self-contained CEL expression, but + // permit some cases where one type is more specific than the other. + return OutputTypeIsAssignable(from, to) || OutputTypeIsAssignable(to, from); +} + +bool HasErrors(const policy_internal::IssueReporter& issues) { + for (const auto& issue : issues.issues()) { + if (issue.severity() == CelPolicyIssue::Severity::kError) { + return true; + } + } + return false; +} + +// Note on lifetime safety: +// +// The output policy will contain references to types that are owned by the +// arena member of this class. This is safe as long as the policy compiler lives +// as long as the output policies. +class PolicyCompiler { + public: + explicit PolicyCompiler(policy_internal::IssueReporter* issues, + std::unique_ptr base_compiler) + : issues_(*issues), base_compiler_(std::move(base_compiler)) {} + + absl::string_view GetSourceDescription() const { + if (src_ == nullptr) { + return ""; + } + return src_->content()->description(); + } + + void AdaptTypeCheckIssues(CelPolicyElementId id, const ValidationResult& r) { + const Source* source = r.GetSource(); + + for (const auto& iss : r.GetIssues()) { + std::optional offset; + if (source != nullptr) { + offset = source->GetPosition(iss.location()); + } + if (offset.has_value()) { + issues_.ReportOffsetIssue(id, offset.value(), + MapSeverity(iss.severity()), iss.message()); + continue; + } + issues_.ReportIssue(id, MapSeverity(iss.severity()), iss.message()); + } + } + + absl::StatusOr CompileOutputBlock( + const cel::OutputBlock& output_block, const Compiler* env) { + CompiledOutputBlock output; + CEL_ASSIGN_OR_RETURN(auto output_validation, + env->Compile(output_block.output().value(), + GetSourceDescription(), &arena_)); + AdaptTypeCheckIssues(output_block.output().id(), output_validation); + + cel::Type result_type = DynType(); + if (output_validation.IsValid()) { + CEL_ASSIGN_OR_RETURN(auto ast, output_validation.ReleaseAst()); + auto root_expr_id = ast->root_expr().id(); + output.output_ast = + EmbeddedAst{output_block.output().id(), std::move(ast)}; + if (auto it = output_validation.GetResolvedTypeMap().find(root_expr_id); + it != output_validation.GetResolvedTypeMap().end()) { + result_type = it->second; + } + } + if (output_block.explanation().has_value()) { + CEL_ASSIGN_OR_RETURN(auto explanation_validation, + env->Compile(output_block.explanation()->value(), + GetSourceDescription(), &arena_)); + AdaptTypeCheckIssues(output_block.explanation()->id(), + explanation_validation); + if (explanation_validation.IsValid()) { + CEL_ASSIGN_OR_RETURN(auto ast, explanation_validation.ReleaseAst()); + if (ast->GetReturnType().primitive() != PrimitiveType::kString) { + issues_.ReportError(output_block.explanation()->id(), + "explanation must evaluate to string"); + } else { + output.explanation_ast = + EmbeddedAst{output_block.explanation()->id(), std::move(ast)}; + } + } + } + output.result_type = result_type; + return output; + } + + absl::Status CompileMatch(const Match& match, const Compiler* env, + CompiledRule* out) { + CompiledMatch c_match; + c_match.id = match.id(); + if (match.condition().has_value()) { + CEL_ASSIGN_OR_RETURN(auto validation, + env->Compile(match.condition()->value(), + GetSourceDescription(), &arena_)); + AdaptTypeCheckIssues(match.condition()->id(), validation); + if (validation.IsValid()) { + CEL_ASSIGN_OR_RETURN(auto ast, validation.ReleaseAst()); + if (ast->GetReturnType().primitive() != PrimitiveType::kBool) { + issues_.ReportError(match.condition()->id(), + "condition must evaluate to bool"); + } + c_match.condition = + EmbeddedAst{match.condition()->id(), std::move(ast)}; + } + } + + if (match.has_output_block()) { + CEL_ASSIGN_OR_RETURN(c_match.production, + CompileOutputBlock(match.output_block(), env)); + } else if (match.has_rule()) { + auto rule = std::make_unique(); + CEL_RETURN_IF_ERROR(CompileRule(match.rule(), env, rule.get())); + c_match.production = std::move(rule); + } else { + issues_.ReportError(match.id(), "match must specify an output or rule"); + } + out->matches.push_back(std::move(c_match)); + return absl::OkStatus(); + } + + absl::Status CompileRule(const Rule& rule, const cel::Compiler* env, + CompiledRule* out) { + out->id = rule.id(); + std::unique_ptr buf; + + absl::flat_hash_set seen_variables; + for (const auto& variable : rule.variables()) { + std::string name(variable.name().value()); + if (!seen_variables.insert(name).second) { + issues_.ReportError( + variable.expression().id(), + absl::StrCat("overlapping identifier for name 'variables.", name, + "'")); + continue; + } + std::string ident = absl::StrCat("variables.", name); + CEL_ASSIGN_OR_RETURN(auto validation, + env->Compile(variable.expression().value(), + GetSourceDescription(), &arena_)); + AdaptTypeCheckIssues(variable.expression().id(), validation); + if (!validation.IsValid()) { + continue; + } + CEL_ASSIGN_OR_RETURN(auto ast, validation.ReleaseAst()); + cel::Type result_type = DynType(); + + if (auto it = validation.GetResolvedTypeMap().find(ast->root_expr().id()); + it != validation.GetResolvedTypeMap().end()) { + result_type = it->second; + } + out->variables.push_back(CompiledVariable{ + ident, + EmbeddedAst{variable.expression().id(), std::move(ast)}, + }); + auto next = env->ToBuilder(); + auto status = next->GetCheckerBuilder().AddOrReplaceVariable( + MakeVariableDecl(ident, result_type)); + if (!status.ok()) { + issues_.ReportError(variable.expression().id(), status.message()); + continue; + } + CEL_ASSIGN_OR_RETURN(buf, next->Build()); + env = buf.get(); + } + + std::optional overall_type; + for (const auto& match : rule.matches()) { + CEL_RETURN_IF_ERROR(CompileMatch(match, env, out)); + if (!overall_type.has_value()) { + overall_type = GetOutputType(out->matches.back().production); + continue; + } + + if (std::optional match_type = + GetOutputType(out->matches.back().production); + match_type.has_value()) { + if (!OutputTypeIsCompatible(*match_type, *overall_type)) { + issues_.ReportError( + match.id(), + absl::StrCat("incompatible output types: block has output type ", + FormatTypeName(*match_type), + ", but previous outputs have type ", + FormatTypeName(*overall_type))); + } + } + } + + out->result_type = overall_type; + return absl::OkStatus(); + } + + absl::Status CompilePolicy(const CelPolicy& policy, + IntermediateCompiledPolicy* out) { + src_ = policy.source(); + out->set_semantics(RuleSemantics::kFirstMatch); + out->set_name(policy.name().value()); + out->set_display_name( + policy.display_name().value_or(ValueString{}).value()); + out->set_description(policy.description().value_or(ValueString{}).value()); + + return CompileRule(policy.rule(), base_compiler_.get(), + &out->mutable_root_rule()); + } + + private: + google::protobuf::Arena arena_; + const CelPolicySource* absl_nullable src_; + policy_internal::IssueReporter& issues_; + std::unique_ptr base_compiler_; +}; + +bool IsExhaustive(const CompiledRule& rule); + +class FirstMatchComposer { + public: + FirstMatchComposer(const IntermediateCompiledPolicy& icp, + const Compiler& compiler, + policy_internal::IssueReporter& issues) + : issues_(issues), icp_(icp), compiler_(compiler) {} + + absl::Status Compose(); + + bool success() const { return ast_ != nullptr; } + + std::unique_ptr ReleaseAst() { return std::move(ast_); } + + private: + using VariableScope = absl::flat_hash_map; + + std::optional ResolvePolicyVariable(absl::string_view reference); + + absl::flat_hash_map ResolveBlockIndexes(const Ast& ast); + + bool CheckMatchStructure(const CompiledRule& rule); + + // Returns true if already optional wrapped. + absl::StatusOr ComposeRule(const CompiledRule& rule, Expr& init, + Expr& insertion_expr); + + // returns true if already optional wrapped. + absl::StatusOr ComposeProduction( + const CompiledRule& rule, const CompiledMatch::Production& production, + Expr& init, Expr& insertion_expr); + + void MapVariables(Ast& ast); + + void ComposeRuleVariables(const CompiledRule& rule, Expr& init, + Expr& insertion_expr); + + policy_internal::IssueReporter& issues_; + OptimizerExprFactory factory_; + const IntermediateCompiledPolicy& icp_; + const Compiler& compiler_; + std::vector scopes_; + bool optionalize_ = false; + std::unique_ptr ast_; +}; + +absl::Status FirstMatchComposer::Compose() { + ABSL_DCHECK(icp_.semantics() == RuleSemantics::kFirstMatch); + + factory_.mutable_ast().mutable_root_expr() = factory_.NewCall( + "cel.@block", factory_.NewList(), factory_.NewUnspecified()); + auto& block_init_list = factory_.mutable_ast() + .mutable_root_expr() + .mutable_call_expr() + .mutable_args()[0]; + auto& insertion_expr = factory_.mutable_ast() + .mutable_root_expr() + .mutable_call_expr() + .mutable_args()[1]; + optionalize_ = !IsExhaustive(icp_.root_rule()); + if (!CheckMatchStructure(icp_.root_rule())) { + return absl::OkStatus(); + } + CEL_ASSIGN_OR_RETURN( + bool optional_wrapped, + ComposeRule(icp_.root_rule(), block_init_list, insertion_expr)); + + if (optional_wrapped != optionalize_) { + return absl::InternalError( + "composition failed to handle non-exhaustive rules"); + } + + CEL_ASSIGN_OR_RETURN(cel::ValidationResult result, + compiler_.GetTypeChecker().Check(factory_.ast())); + if (!result.IsValid()) { + for (const auto& iss : result.GetIssues()) { + issues_.ReportError(icp_.root_rule().id, iss.message()); + } + return absl::OkStatus(); + } + + CEL_ASSIGN_OR_RETURN(ast_, result.ReleaseAst()); + + return absl::OkStatus(); +} + +bool IsTriviallyTrueCondition(const CompiledMatch& match) { + if (!match.condition.has_value() || match.condition->ast == nullptr) { + return true; + } + const cel::Expr& expr = match.condition->ast->root_expr(); + if (expr.has_const_expr()) { + const cel::Constant& const_expr = expr.const_expr(); + if (const_expr.has_bool_value() && const_expr.bool_value()) { + return true; + } + } + return false; +} + +bool IsExhaustive(const CompiledRule& rule); + +bool IsExhaustive(const CompiledMatch& match) { + if (std::holds_alternative(match.production)) { + return true; + } + + const auto* nested_rule_ptr = + std::get_if>(&match.production); + ABSL_DCHECK(nested_rule_ptr != nullptr); + const CompiledRule& nested_rule = **nested_rule_ptr; + return IsExhaustive(nested_rule); +} + +bool IsExhaustive(const CompiledRule& rule) { + if (rule.matches.empty()) { + // Validation should fail, but generalization would be false. + return false; + } + bool has_default = false; + for (const auto& match : rule.matches) { + if (IsTriviallyTrueCondition(match) && IsExhaustive(match)) { + // If this isn't the last match in the rule, it should get flagged + // during validation since it means there are trivially unreachable + // matches. + has_default = true; + } + if (!IsTriviallyTrueCondition(match) && !IsExhaustive(match)) { + // There is a nested rule that might return an optional.none(). + return false; + } + } + // Otherwise, everything in this branch is exhaustive so we can defer + // wrapping. + return has_default; +} + +bool FirstMatchComposer::CheckMatchStructure(const CompiledRule& rule) { + if (rule.matches.empty()) { + issues_.ReportError(rule.id, "rule does not specify match conditions"); + return false; + } + + bool valid = true; + bool seen_trivially_true = false; + + for (const auto& match : rule.matches) { + if (seen_trivially_true) { + if (std::holds_alternative(match.production)) { + issues_.ReportError(match.id, "match creates unreachable outputs"); + } else if (std::holds_alternative>( + match.production)) { + issues_.ReportError(match.id, "rule creates unreachable outputs"); + } + valid = false; + } + + if (IsTriviallyTrueCondition(match) && IsExhaustive(match)) { + seen_trivially_true = true; + } + + if (auto* nested_rule = + std::get_if>(&match.production); + nested_rule != nullptr) { + ABSL_DCHECK(*nested_rule != nullptr); + if (!CheckMatchStructure(**nested_rule)) { + valid = false; + } + } + } + + return valid; +} + +std::optional FirstMatchComposer::ResolvePolicyVariable( + absl::string_view reference) { + for (auto scope_iter = scopes_.rbegin(); scope_iter != scopes_.rend(); + ++scope_iter) { + if (auto it = scope_iter->find(reference); it != scope_iter->end()) { + return it->second; + } + } + return std::nullopt; +} + +class IndexRewrite : public AstRewriterBase { + public: + explicit IndexRewrite(absl::flat_hash_map expr_id_to_index, + OptimizerExprFactory& factory) + : expr_id_to_index_(std::move(expr_id_to_index)), factory_(factory) {} + + bool PreVisitRewrite(Expr& e) override { + if (auto it = expr_id_to_index_.find(e.id()); + it != expr_id_to_index_.end()) { + e.mutable_ident_expr().set_name(absl::StrCat("@index", it->second)); + factory_.RecordReplacement(e.id(), e); + return true; + } + return false; + } + + private: + absl::flat_hash_map expr_id_to_index_; + OptimizerExprFactory& factory_; +}; + +absl::StatusOr FirstMatchComposer::ComposeRule(const CompiledRule& rule, + Expr& init, + Expr& insertion_expr) { + scopes_.emplace_back(); + auto pop_scope = absl::MakeCleanup([this]() { scopes_.pop_back(); }); + ComposeRuleVariables(rule, init, insertion_expr); + Expr* insertion_point = &insertion_expr; + const bool has_default = IsTriviallyTrueCondition(rule.matches.back()); + const bool needs_wrap = !IsExhaustive(rule); + size_t end = rule.matches.size() - (has_default ? 1 : 0); + for (size_t i = 0; i < end; i++) { + const auto& match = rule.matches[i]; + if (IsTriviallyTrueCondition(match) && IsExhaustive(match)) { + return absl::InternalError("detected unreachable match after validation"); + } + + Expr production; + CEL_ASSIGN_OR_RETURN( + bool is_wrapped, + ComposeProduction(rule, match.production, init, production)); + if (needs_wrap && !is_wrapped) { + production = factory_.NewCall("optional.of", std::move(production)); + } + + if (!IsTriviallyTrueCondition(match)) { + Ast condition = *match.condition->ast; + MapVariables(condition); + factory_.StartCopyContext(); + auto copy = factory_.Copy(condition.root_expr()); + auto source_info = factory_.RemapSourceInfo(condition.source_info()); + factory_.MergeSourceInfo(source_info); + *insertion_point = factory_.NewCall("_?_:_", std::move(copy)); + insertion_point->mutable_call_expr().mutable_args().push_back( + std::move(production)); + ABSL_DCHECK(!(!needs_wrap && is_wrapped)) + << "unexpected wrapping in exhaustive policy."; + insertion_point = &insertion_point->mutable_call_expr().add_args(); + continue; + } + + if (!is_wrapped) { + return absl::InternalError( + "composition failed. expected optional wrapped rule but got a plain " + "value"); + } + auto fn = needs_wrap ? "or" : "orValue"; + *insertion_point = factory_.NewMemberCall(fn, std::move(production)); + insertion_point = &insertion_point->mutable_call_expr().add_args(); + } + + if (has_default) { + const auto& match = rule.matches.back(); + Expr production; + CEL_ASSIGN_OR_RETURN( + bool is_wrapped, + ComposeProduction(rule, match.production, init, production)); + if (needs_wrap && !is_wrapped) { + production = factory_.NewCall("optional.of", std::move(production)); + } + *insertion_point = std::move(production); + ABSL_DCHECK(!(!needs_wrap && is_wrapped)) + << "unexpected wrapping in exhaustive policy."; + + return needs_wrap; + } + + // Otherwise, we fell through a non-exhaustive rule. + *insertion_point = factory_.NewCall("optional.none"); + return true; +} + +absl::StatusOr FirstMatchComposer::ComposeProduction( + const CompiledRule& rule, const CompiledMatch::Production& production, + Expr& init, Expr& insertion_expr) { + if (auto* nested_rule = + std::get_if>(&production); + nested_rule != nullptr) { + return ComposeRule(**nested_rule, init, insertion_expr); + } + auto* output = std::get_if(&production); + if (output == nullptr) { + return absl::InternalError("unexpected rule production type"); + } + const EmbeddedAst& output_ast = output->output_ast; + Ast ast = *output_ast.ast; + MapVariables(ast); + factory_.StartCopyContext(); + Expr to_insert = factory_.Copy(ast.root_expr()); + auto source_info = factory_.RemapSourceInfo(ast.source_info()); + factory_.MergeSourceInfo(source_info); + insertion_expr = std::move(to_insert); + + return false; +} + +absl::flat_hash_map FirstMatchComposer::ResolveBlockIndexes( + const Ast& ast) { + absl::flat_hash_map out; + for (auto it = ast.reference_map().begin(); it != ast.reference_map().end(); + it++) { + const Reference& ref = it->second; + if (!it->second.overload_id().empty()) { + continue; + } + if (!absl::StartsWith(ref.name(), "variable")) { + continue; + } + if (auto index = ResolvePolicyVariable(ref.name()); index.has_value()) { + out[it->first] = *index; + } + } + return out; +} + +void FirstMatchComposer::MapVariables(Ast& ast) { + absl::flat_hash_map edit_map = ResolveBlockIndexes(ast); + IndexRewrite rewriter(std::move(edit_map), factory_); + AstRewrite(ast.mutable_root_expr(), rewriter); +} + +void FirstMatchComposer::ComposeRuleVariables(const CompiledRule& rule, + Expr& init, + Expr& insertion_expr) { + for (const auto& variable : rule.variables) { + Ast ast = *variable.ast.ast; + MapVariables(ast); + factory_.StartCopyContext(); + auto insertion = factory_.Copy(ast.root_expr()); + // TODO(b/506179116): apply the position offsets here. + auto info = factory_.RemapSourceInfo(ast.source_info()); + ABSL_DCHECK(init.has_list_expr()); + int index = init.mutable_list_expr().elements().size(); + init.mutable_list_expr().mutable_elements().push_back( + factory_.NewListElement(std::move(insertion))); + scopes_.back()[variable.ident] = index; + } +} + +bool HasComprehensionParent(const NavigableAstNode& node) { + const NavigableAstNode* curr = &node; + while (curr != nullptr) { + if (curr->node_kind() == NodeKind::kComprehension) { + return true; + } + curr = curr->parent(); + } + return false; +} + +// Unnester implementation. +class Unnester { + public: + Unnester(Ast ast, int height, policy_internal::IssueReporter& issues) + : factory_(std::move(ast)), height_(height), issues_(issues) {} + + // Run the unnesting. + // The class cannot be reused after this is called. + absl::StatusOr Unnest() { + if (height_ > 0) { + CEL_RETURN_IF_ERROR(Slice()); + } + CEL_RETURN_IF_ERROR(Cleanup()); + return std::move(factory_.mutable_ast()); + } + + private: + // The core unnest routine. + absl::Status Slice(); + // Fixup the AST post-unnesting. + absl::Status Cleanup(); + + void ReportErrorAtId(int64_t id, absl::string_view message); + + OptimizerExprFactory factory_; + int height_; + policy_internal::IssueReporter& issues_; +}; + +class UnnestRewriter : public AstRewriterBase { + public: + explicit UnnestRewriter(OptimizerExprFactory& f, Expr& block_list_expr, + absl::Span cuts) + : factory_(f), cuts_(cuts), block_list_expr_(block_list_expr) {} + + bool PostVisitRewrite(Expr& expr) override { + using std::swap; + // Post order so we always see children before parents. + // No need to copy metadata since we're only moving exprs or minting + // new ones. + if (absl::c_contains(cuts_, expr.id())) { + size_t idx = block_list_expr_.list_expr().elements().size(); + Expr value = factory_.NewIdent(absl::StrCat("@index", idx)); + factory_.RecordReplacement(expr.id(), value, /*keep_metadata=*/true); + swap(value, expr); + block_list_expr_.mutable_list_expr().mutable_elements().push_back( + factory_.NewListElement(std::move(value))); + return true; + } + return false; + } + + private: + OptimizerExprFactory& factory_; + absl::Span cuts_; + Expr& block_list_expr_; +}; + +absl::Status Unnester::Slice() { + Expr& root = factory_.mutable_ast().mutable_root_expr(); + if (root.call_expr().function() != kCelBlock || + root.call_expr().args().size() != 2 || + !root.call_expr().args()[0].has_list_expr()) { + return absl::InternalError("malformed AST detected during unnesting"); + } + // Two passes, we identify the slice points (bottom up), then cut + // and paste the leaves into the block list. + NavigableAst nav_ast = NavigableAst::Build(factory_.ast().root_expr()); + + ABSL_DCHECK(nav_ast.IdsAreUnique()); + bool can_cut = true; + std::vector cuts; + for (const NavigableAstNode& node : nav_ast.Root().DescendantsPostorder()) { + // Subsequent cuts will be height_ + 1 in the block, indices. Within the + // error margin we specified. + if (node.height() % height_ == 0) { + if (HasComprehensionParent(node)) { + ReportErrorAtId( + node.expr()->id(), + absl::StrCat( + "cannot unnest AST due to comprehension. cannot accommodate " + "height limit of ", + height_)); + can_cut = false; + continue; + } + if (&node == &nav_ast.Root()) { + // If evenly divisible by height, don't cut since it will net a taller + // AST. + continue; + } + cuts.push_back(node.expr()->id()); + } + } + + if (!can_cut || cuts.empty()) { + return absl::OkStatus(); + } + + Expr& block_list_expr = root.mutable_call_expr().mutable_args()[0]; + Expr& insertion_expr = root.mutable_call_expr().mutable_args()[1]; + + UnnestRewriter rewriter(factory_, block_list_expr, cuts); + AstRewrite(insertion_expr, rewriter); + + return absl::OkStatus(); +} + +absl::Status Unnester::Cleanup() { + using std::swap; + + const auto& ast = factory_.ast(); + if (ast.root_expr().call_expr().function() != kCelBlock || + ast.root_expr().call_expr().args().size() != 2 || + !ast.root_expr().call_expr().args()[0].has_list_expr()) { + return absl::InternalError("malformed AST detected during unnesting"); + } + if (ast.root_expr().call_expr().args()[0].list_expr().elements().empty()) { + Expr value = std::move(factory_.mutable_ast() + .mutable_root_expr() + .mutable_call_expr() + .mutable_args()[1]); + factory_.mutable_ast().mutable_root_expr() = std::move(value); + } + + return absl::OkStatus(); +} + +void Unnester::ReportErrorAtId(int64_t id, absl::string_view message) { + int32_t position = 0; + auto it = factory_.ast().source_info().positions().find(id); + if (it != factory_.ast().source_info().positions().end()) { + position = it->second; + } + issues_.ReportError(-1, position, message); +} +} // namespace + +// Compiles a CEL policy using the provided CEL compiler as a base environment. +absl::StatusOr CompilePolicy( + const Compiler& compiler, const CelPolicy& policy, + const CompilePolicyOptions& options) { + policy_internal::IssueReporter issues; + if (options.unnesting_height_limit != 0 && + options.unnesting_height_limit < 2) { + return absl::InvalidArgumentError( + "unnesting_height_limit must be at least 2"); + } + auto builder = compiler.ToBuilder(); + ExpressionContainer cont; + for (const auto& import : policy.imports()) { + auto status = cont.AddAbbreviation(import.name().value()); + if (!status.ok()) { + issues.ReportError( + import.name().id(), + absl::StrCat("'", import.name().value(), "': ", status.message())); + } + } + + builder->GetCheckerBuilder().SetExpressionContainer(cont); + CEL_ASSIGN_OR_RETURN(auto base_compiler, builder->Build()); + + PolicyCompiler policy_compiler(&issues, std::move(base_compiler)); + + IntermediateCompiledPolicy icp; + CEL_RETURN_IF_ERROR(policy_compiler.CompilePolicy(policy, &icp)); + + if (HasErrors(issues)) { + return CelPolicyValidationResult(issues.ReleaseIssues(), + policy.source_ptr()); + } + + CEL_ASSIGN_OR_RETURN(base_compiler, builder->Build()); + switch (icp.semantics()) { + case RuleSemantics::kFirstMatch: { + FirstMatchComposer composer(icp, *base_compiler, issues); + CEL_RETURN_IF_ERROR(composer.Compose()); + if (!composer.success()) { + return CelPolicyValidationResult(issues.ReleaseIssues(), + policy.source_ptr()); + } + + auto ast = composer.ReleaseAst(); + Unnester unnester(std::move(*ast), options.unnesting_height_limit, + issues); + CEL_ASSIGN_OR_RETURN(Ast unnested_ast, unnester.Unnest()); + + if (HasErrors(issues)) { + return CelPolicyValidationResult(issues.ReleaseIssues(), + policy.source_ptr()); + } + + return CelPolicyValidationResult( + std::make_unique(std::move(unnested_ast)), {}, + policy.source_ptr()); + } + default: + return absl::UnimplementedError( + absl::StrCat("Unsupported RuleSemantics: ", icp.semantics())); + } +} + +} // namespace cel diff --git a/policy/compiler.h b/policy/compiler.h new file mode 100644 index 000000000..0187bd1a2 --- /dev/null +++ b/policy/compiler.h @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_COMPILER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_COMPILER_H_ + +#include "absl/status/statusor.h" +#include "compiler/compiler.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_validation_result.h" + +namespace cel { + +struct CompilePolicyOptions { + // If greater than 0, the compiler will attempt to unnest rule branches + // at the specified height. The overall height of the final AST may exceed + // this by a small, fixed margin. + // + // To avoid slicing comprehensions, subexpressions within comprehensions + // are not eligible for unnesting. If the height limit cannot be accommodated, + // an error with code InvalidArgument is returned. + // + // If the AST is converted to proto, even relatively low levels of nesting + // can cause problems in serialization/deserialization. This does not apply + // if the AST is used directly by the runtime. + int unnesting_height_limit = 0; +}; + +// Compiles a CEL policy using the provided CEL compiler as a base environment. +// +// TODO(b/506179116): Implementation in progress. Functionally complete, +// but errors are not consistent with other implementations. +absl::StatusOr CompilePolicy( + const Compiler& compiler, const CelPolicy& policy, + const CompilePolicyOptions& options = {}); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_COMPILER_H_ diff --git a/policy/compiler_test.cc b/policy/compiler_test.cc new file mode 100644 index 000000000..8db494b45 --- /dev/null +++ b/policy/compiler_test.cc @@ -0,0 +1,946 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/compiler.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/decl.h" +#include "common/navigable_ast.h" +#include "common/source.h" +#include "common/type.h" +#include "common/types/message_type.h" +#include "common/value.h" +#include "common/value_testing.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/optional.h" +#include "compiler/standard_library.h" +#include "extensions/bindings_ext.h" +#include "internal/runfiles.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_validation_result.h" +#include "policy/yaml_policy_parser.h" +#include "runtime/activation.h" +#include "runtime/optional_types.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::cel::test::IntValueIs; +using ::cel::test::OptionalValueIs; +using ::cel::test::OptionalValueIsEmpty; +using ::cel::test::StringValueIs; +using ::cel::test::ValueMatcher; + +constexpr absl::string_view kTestPolicyFilePath = +"_main/policy/testdata/cel_policy.yaml"; + +absl::StatusOr> BuildTestCompiler() { + CompilerOptions opts; + opts.adapt_parser_errors = true; + opts.parser_options.enable_optional_syntax = true; + CEL_ASSIGN_OR_RETURN( + auto builder, + NewCompilerBuilder(internal::GetSharedTestingDescriptorPool(), opts)); + + CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::StandardCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::OptionalCompilerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(cel::extensions::BindingsCompilerLibrary())); + + CEL_RETURN_IF_ERROR(builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("x", IntType()))); + CEL_RETURN_IF_ERROR(builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("y", IntType()))); + + const google::protobuf::Descriptor* descriptor = + cel::internal::GetSharedTestingDescriptorPool()->FindMessageTypeByName( + "cel.expr.conformance.proto3.TestAllTypes"); + if (descriptor == nullptr) { + return absl::InternalError("Failed to find TestAllTypes descriptor"); + } + CEL_RETURN_IF_ERROR(builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("spec", cel::MessageType(descriptor)))); + + return builder->Build(); +} + +absl::StatusOr> ParsePolicyFromYaml( + absl::string_view yaml_content) { + CEL_ASSIGN_OR_RETURN(auto source, cel::NewSource(yaml_content, "test.yaml")); + + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + CEL_ASSIGN_OR_RETURN(auto parse_result, + cel::ParseYamlCelPolicy(policy_source)); + + if (!parse_result.IsValid()) { + return absl::InvalidArgumentError("Invalid policy YAML structure"); + } + return parse_result.ReleasePolicy(); +} + +TEST(CompilerTest, SmokeTest) { + std::string contents; + std::string test_file = + cel::internal::ResolveRunfilesPath(kTestPolicyFilePath); + auto read_status = cel::internal::GetFileContents(test_file, &contents); + ASSERT_THAT(read_status, IsOk()); + + auto source_or = cel::NewSource(contents, "cel_policy.yaml"); + ASSERT_THAT(source_or.status(), IsOk()); + auto source = *std::move(source_or); + + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + auto parse_result_or = cel::ParseYamlCelPolicy(policy_source); + ASSERT_THAT(parse_result_or.status(), IsOk()); + auto parse_result = *std::move(parse_result_or); + + ASSERT_TRUE(parse_result.IsValid()); + const CelPolicy* policy = parse_result.GetPolicy(); + + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); +} + +TEST(CompilerTest, VariableOutOfScopeReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: variables.non_existent == 10 + output: '"error"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("undeclared reference")); +} + +TEST(CompilerTest, ConditionNotBoolReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: 10 + output: '"error"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("condition must evaluate to bool")); +} + +TEST(CompilerTest, InvalidOutputExpressionReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true + output: undeclared_var +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("undeclared reference")); +} + +TEST(CompilerTest, UnreachableMatchAfterTriviallyTrueCondition) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true + output: '"first"' + - condition: true + output: '"second"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("match creates unreachable outputs")); +} + +TEST(CompilerTest, UnreachableMatchAfterUnconditionalExhaustiveSubRule) { + absl::string_view yaml = R"yaml( +name: dead_branch +rule: + match: + - rule: + match: + - output: 1 + - output: 2 +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("match creates unreachable outputs")); +} + +TEST(CompilerTest, RuleWithoutMatchesReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("rule does not specify match conditions")); +} + +TEST(CompilerTest, ExhaustivePolicyCompiles) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + variables: + - name: test_var + expression: 10 + match: + - condition: variables.test_var > 15 + output: '"greater than 15"' + - condition: variables.test_var > 5 + output: '"greater than 5"' + - output: '"default"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); + EXPECT_TRUE(result.GetAst()->is_checked()); +} + +TEST(CompilerTest, NonExhaustivePolicyCompiles) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + variables: + - name: test_var + expression: 10 + match: + - condition: variables.test_var > 5 + output: '"greater than 5"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); +} + +TEST(CompilerTest, PolicyReferencesEnvInput) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: spec.single_int32 > 10 + output: '"greater than 10"' + - output: '"default"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); + EXPECT_TRUE(result.GetAst()->is_checked()); +} + +struct EvaluationTestCase { + std::string name; + std::string yaml_policy; + struct Input { + int64_t x; + int64_t y; + } input; + ValueMatcher expected_result_matcher; +}; + +class PolicyEvaluationTest : public testing::TestWithParam { +}; + +TEST_P(PolicyEvaluationTest, Evaluate) { + const auto& test_case = GetParam(); + + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(test_case.yaml_policy)); + ASSERT_OK_AND_ASSIGN(auto validation_result, + CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + // Set up runtime + cel::RuntimeOptions opts; + opts.enable_qualified_type_identifiers = true; + ASSERT_OK_AND_ASSIGN( + cel::RuntimeBuilder rt_builder, + cel::CreateStandardRuntimeBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + ASSERT_THAT(cel::extensions::EnableOptionalTypes(rt_builder), IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(rt_builder).Build()); + + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); + + // Set up activation + cel::Activation activation; + activation.InsertOrAssignValue("x", cel::IntValue(test_case.input.x)); + activation.InsertOrAssignValue("y", cel::IntValue(test_case.input.y)); + + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + + EXPECT_THAT(result, test_case.expected_result_matcher); +} + +constexpr absl::string_view kEvalPolicyYaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: x > 10 && y > 10 + output: '"both greater than 10"' + - condition: x > 10 + output: '"x greater than 10"' + - condition: y > 10 + output: '"y greater than 10"' + - output: '"default"' +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + PolicyEvaluationTest, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "BothGreaterThan10", + .yaml_policy = std::string(kEvalPolicyYaml), + .input = {.x = 15, .y = 15}, + .expected_result_matcher = StringValueIs("both greater than 10"), + }, + EvaluationTestCase{ + .name = "XGreaterThan10", + .yaml_policy = std::string(kEvalPolicyYaml), + .input = {.x = 15, .y = 5}, + .expected_result_matcher = StringValueIs("x greater than 10"), + }, + EvaluationTestCase{ + .name = "YGreaterThan10", + .yaml_policy = std::string(kEvalPolicyYaml), + .input = {.x = 5, .y = 15}, + .expected_result_matcher = StringValueIs("y greater than 10"), + }, + EvaluationTestCase{ + .name = "Default", + .yaml_policy = std::string(kEvalPolicyYaml), + .input = {.x = 5, .y = 5}, + .expected_result_matcher = StringValueIs("default"), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kNonExhaustivePolicyYaml = R"yaml( +name: nested_rule4 +rule: + match: + - condition: x > 0 + rule: + match: + - condition: x < 3 + output: 1 + - condition: x < 5 + output: 2 + - condition: x < 0 + rule: + match: + - condition: x > -2 + output: 3 + - condition: x > -4 + output: 4 + - output: 5 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + NonExhaustivePolicyEvaluation, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals0_FallthroughTopLevel", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = 0, .y = 0}, + .expected_result_matcher = OptionalValueIsEmpty(), + }, + EvaluationTestCase{ + .name = "XEquals2_MatchesFirstNested", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = 2, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(1)), + }, + EvaluationTestCase{ + .name = "XEquals6_FallthroughNested", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = 6, .y = 0}, + .expected_result_matcher = OptionalValueIsEmpty(), + }, + EvaluationTestCase{ + .name = "XEqualsMinus1_MatchesMinus2", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(3)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus3_MatchesMinus4", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = -3, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(4)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus5_MatchesDefault", + .yaml_policy = std::string(kNonExhaustivePolicyYaml), + .input = {.x = -5, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(5)), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kNestedVariablePolicyYaml = R"yaml( +name: nested_rule4 +rule: + variables: + - name: i + expression: "1" + - name: j + expression: "2" + match: + - condition: x > 0 + rule: + variables: + - name: k + expression: "3" + match: + - output: "variables.i + variables.j + variables.k" + - condition: x < 0 + rule: + variables: + - name: j + expression: "5" + - name: k + expression: "4" + match: + - output: "variables.i + variables.j + variables.k" + - output: "variables.i + variables.j" +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + NestedVariablePolicyEvaluation, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XGreaterThan0", + .yaml_policy = std::string(kNestedVariablePolicyYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = IntValueIs(6), + }, + EvaluationTestCase{ + .name = "XLessThan0", + .yaml_policy = std::string(kNestedVariablePolicyYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = IntValueIs(10), + }, + EvaluationTestCase{ + .name = "XEquals0", + .yaml_policy = std::string(kNestedVariablePolicyYaml), + .input = {.x = 0, .y = 0}, + .expected_result_matcher = IntValueIs(3), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view + kOptionalChainingUnconditionalSubRuleOptionalParentYaml = R"yaml( +name: optional_chaining +rule: + match: + - rule: + id: r2 + match: + - condition: x > 0 + output: 1 + - output: 2 + condition: x < 0 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + OptionalChainingUnconditionalSubRuleOptionalParent, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = std::string( + kOptionalChainingUnconditionalSubRuleOptionalParentYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(1)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus1", + .yaml_policy = std::string( + kOptionalChainingUnconditionalSubRuleOptionalParentYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(2)), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kOptionalChainingUnconditionalSubRuleYaml = R"yaml( +name: optional_chaining +rule: + id: r1 + match: + - rule: + id: r2 + match: + - condition: x > 0 + output: 1 + - output: 2 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + OptionalChainingUnconditionalSubRule, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = + std::string(kOptionalChainingUnconditionalSubRuleYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = IntValueIs(1), + }, + EvaluationTestCase{ + .name = "XEqualsMinus1", + .yaml_policy = + std::string(kOptionalChainingUnconditionalSubRuleYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = IntValueIs(2), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kOptionalChainingUnconditionalComplexYaml = R"yaml( +name: optional_chaining +rule: + match: + - condition: x > 0 + rule: + match: + - rule: + match: + - condition: x == 1 + output: 1 + - output: 2 + - rule: + match: + - condition: x == -1 + output: 3 + - condition: x == -2 + output: 4 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + OptionalChainingUnconditionalComplex, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = + std::string(kOptionalChainingUnconditionalComplexYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(1)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus1", + .yaml_policy = + std::string(kOptionalChainingUnconditionalComplexYaml), + .input = {.x = -1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(3)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus2", + .yaml_policy = + std::string(kOptionalChainingUnconditionalComplexYaml), + .input = {.x = -2, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(4)), + }, + EvaluationTestCase{ + .name = "XEqualsMinus3", + .yaml_policy = + std::string(kOptionalChainingUnconditionalComplexYaml), + .input = {.x = -3, .y = 0}, + .expected_result_matcher = OptionalValueIsEmpty(), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kUnconditionalExhaustiveSubRuleAsLastMatchYaml = + R"yaml( +name: exhaustive_unconditional_subrule +rule: + match: + - condition: x > 0 + output: 1 + - rule: + match: + - condition: y > 0 + output: 2 + - output: 3 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + UnconditionalExhaustiveSubRuleAsLastMatch, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = + std::string(kUnconditionalExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = IntValueIs(1), + }, + EvaluationTestCase{ + .name = "XEquals0_YEquals1", + .yaml_policy = + std::string(kUnconditionalExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 0, .y = 1}, + .expected_result_matcher = IntValueIs(2), + }, + EvaluationTestCase{ + .name = "XEquals0_YEquals0", + .yaml_policy = + std::string(kUnconditionalExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 0, .y = 0}, + .expected_result_matcher = IntValueIs(3), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +constexpr absl::string_view kUnconditionalNonExhaustiveSubRuleAsLastMatchYaml = + R"yaml( +name: non_exhaustive_unconditional_subrule +rule: + match: + - condition: x > 0 + output: 1 + - rule: + match: + - condition: y > 0 + output: 2 +)yaml"; + +INSTANTIATE_TEST_SUITE_P( + UnconditionalNonExhaustiveSubRuleAsLastMatch, PolicyEvaluationTest, + testing::Values( + EvaluationTestCase{ + .name = "XEquals1", + .yaml_policy = + std::string(kUnconditionalNonExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 1, .y = 0}, + .expected_result_matcher = OptionalValueIs(IntValueIs(1)), + }, + EvaluationTestCase{ + .name = "XEquals0_YEquals1", + .yaml_policy = + std::string(kUnconditionalNonExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 0, .y = 1}, + .expected_result_matcher = OptionalValueIs(IntValueIs(2)), + }, + EvaluationTestCase{ + .name = "XEquals0_YEquals0", + .yaml_policy = + std::string(kUnconditionalNonExhaustiveSubRuleAsLastMatchYaml), + .input = {.x = 0, .y = 0}, + .expected_result_matcher = OptionalValueIsEmpty(), + }), + [](const testing::TestParamInfo& info) { + return info.param.name; + }); + +TEST(CompilerTest, ImportsAndAbbreviations) { + absl::string_view yaml = R"yaml( +name: imports_test +imports: + - name: cel.expr.conformance.proto3.TestAllTypes +rule: + match: + - condition: 'spec == TestAllTypes{single_int32: 10}' + output: '"matched"' + - output: '"default"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + auto ast_or = CompilePolicy(*compiler, *policy); + ASSERT_THAT(ast_or, IsOk()); +} + +TEST(CompilerTest, MatchWithoutProductionReportsError) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("match must specify an output or rule")); +} + +int GetAstHeight(const cel::Ast& ast) { + auto nav_ast = cel::NavigableAst::Build(ast.root_expr()); + return nav_ast.Root().height(); +} + +TEST(CompilerTest, UnnestHeightValidation) { + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true + output: '"ok"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 1; + auto status_or = CompilePolicy(*compiler, *policy, options); + EXPECT_THAT(status_or.status(), + StatusIs(absl::StatusCode::kInvalidArgument, + testing::HasSubstr( + "unnesting_height_limit must be at least 2"))); + + options.unnesting_height_limit = 2; + EXPECT_THAT(CompilePolicy(*compiler, *policy, options), IsOk()); +} + +constexpr absl::string_view kDeepPolicyYaml = R"yaml( +name: deep_policy +rule: + match: + - condition: x > 0 + rule: + match: + - condition: x > 1 + rule: + match: + - condition: x > 2 + rule: + match: + - condition: x > 3 + rule: + match: + - condition: x > 4 + rule: + match: + - condition: x > 5 + output: 6 + - output: 5 + - output: 4 + - output: 3 + - output: 2 + - output: 1 + - output: 0 +)yaml"; + +TEST(CompilerTest, UnnestHeightReduction) { + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(kDeepPolicyYaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + // Compile without unnesting + CompilePolicyOptions options_no_unnest; + options_no_unnest.unnesting_height_limit = 0; + ASSERT_OK_AND_ASSIGN(auto result_no_unnest, + CompilePolicy(*compiler, *policy, options_no_unnest)); + ASSERT_TRUE(result_no_unnest.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast_no_unnest, result_no_unnest.ReleaseAst()); + int height_no_unnest = GetAstHeight(*ast_no_unnest); + + CompilePolicyOptions options_unnest; + options_unnest.unnesting_height_limit = 2; + ASSERT_OK_AND_ASSIGN(auto result_unnest, + CompilePolicy(*compiler, *policy, options_unnest)); + ASSERT_TRUE(result_unnest.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast_unnest, result_unnest.ReleaseAst()); + int height_unnest = GetAstHeight(*ast_unnest); + + EXPECT_EQ(height_no_unnest, 8); + EXPECT_EQ(height_unnest, 5); + EXPECT_LT(height_unnest, height_no_unnest); +} + +TEST(CompilerTest, UnnestComprehensionFailure) { + absl::string_view yaml = R"yaml( +name: comprehension_policy +rule: + match: + - condition: x > 0 + rule: + match: + - condition: "[1, 2].all(i, i > x)" + output: 1 + - output: 2 + - output: 0 +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 2; + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy, options)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT(result.FormatIssues(), + testing::HasSubstr("cannot unnest AST due to comprehension")); +} + +struct UnnestEvaluationTestCase { + std::string name; + int64_t x; + ValueMatcher expected; +}; + +class UnnestedDeepPolicyEvaluationTest + : public testing::TestWithParam {}; + +TEST_P(UnnestedDeepPolicyEvaluationTest, Evaluate) { + const auto& tc = GetParam(); + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(kDeepPolicyYaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 2; + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy, options)); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, result.ReleaseAst()); + + // Set up runtime + cel::RuntimeOptions opts; + opts.enable_qualified_type_identifiers = true; + ASSERT_OK_AND_ASSIGN( + cel::RuntimeBuilder rt_builder, + cel::CreateStandardRuntimeBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + ASSERT_THAT(cel::extensions::EnableOptionalTypes(rt_builder), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + std::move(rt_builder).Build()); + + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); + + cel::Activation activation; + activation.InsertOrAssignValue("x", cel::IntValue(tc.x)); + google::protobuf::Arena arena; + + ASSERT_OK_AND_ASSIGN(cel::Value res, program->Evaluate(&arena, activation)); + + EXPECT_THAT(res, tc.expected); +} + +INSTANTIATE_TEST_SUITE_P( + UnnestedDeepPolicyEvaluation, UnnestedDeepPolicyEvaluationTest, + testing::Values(UnnestEvaluationTestCase{"XEquals6", 6, IntValueIs(6)}, + UnnestEvaluationTestCase{"XEquals5", 5, IntValueIs(5)}, + UnnestEvaluationTestCase{"XEquals4", 4, IntValueIs(4)}, + UnnestEvaluationTestCase{"XEquals3", 3, IntValueIs(3)}, + UnnestEvaluationTestCase{"XEquals2", 2, IntValueIs(2)}, + UnnestEvaluationTestCase{"XEquals1", 1, IntValueIs(1)}, + UnnestEvaluationTestCase{"XEquals0", 0, IntValueIs(0)}, + UnnestEvaluationTestCase{"XEqualsMinus1", -1, + IntValueIs(0)}), + [](const testing::TestParamInfo< + UnnestedDeepPolicyEvaluationTest::ParamType>& info) { + return info.param.name; + }); + +TEST(CompilerTest, UnnestCleanupRunsWhenDisabled) { + // A policy without variables and without nesting. + absl::string_view yaml = R"yaml( +name: cel_policy +rule: + id: test_rule + match: + - condition: true + output: '"ok"' +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + + CompilePolicyOptions options; + options.unnesting_height_limit = 0; // Disabled + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy, options)); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, result.ReleaseAst()); + + // If cleanup ran, it should have optimized away the trivial `cel.@block`. + // So the root expression should NOT be a call to `cel.@block`. + // It should be just the constant `"ok"`. + auto nav_ast = cel::NavigableAst::Build(ast->root_expr()); + EXPECT_FALSE(nav_ast.Root().expr()->has_call_expr() && + nav_ast.Root().expr()->call_expr().function() == "cel.@block"); + EXPECT_TRUE(nav_ast.Root().expr()->has_const_expr()); +} +} // namespace +} // namespace cel diff --git a/policy/internal/BUILD b/policy/internal/BUILD new file mode 100644 index 000000000..30f43d431 --- /dev/null +++ b/policy/internal/BUILD @@ -0,0 +1,68 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "issue_reporter", + srcs = ["issue_reporter.cc"], + hdrs = ["issue_reporter.h"], + deps = [ + "//common:source", + "//policy:cel_policy", + "//policy:cel_policy_parser", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "optimizer_expr_factory", + srcs = ["optimizer_expr_factory.cc"], + hdrs = ["optimizer_expr_factory.h"], + deps = [ + "//common:ast", + "//common:ast_rewrite", + "//common:ast_traverse", + "//common:ast_visitor_base", + "//common:constant", + "//common:expr", + "//common:expr_factory", + "//common:source", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", + ], +) + +cc_test( + name = "optimizer_expr_factory_test", + srcs = ["optimizer_expr_factory_test.cc"], + deps = [ + ":optimizer_expr_factory", + "//common:ast", + "//common:ast_proto", + "//common:ast_rewrite", + "//common:decl", + "//common:expr", + "//common:expr_factory", + "//common:source", + "//common:type", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", + "//internal:status_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//testutil:expr_printer", + "//tools:cel_unparser", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + ], +) diff --git a/policy/internal/issue_reporter.cc b/policy/internal/issue_reporter.cc new file mode 100644 index 000000000..944e687d6 --- /dev/null +++ b/policy/internal/issue_reporter.cc @@ -0,0 +1,45 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/issue_reporter.h" + +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "policy/cel_policy.h" + +namespace cel::policy_internal { + +void IssueReporter::ReportIssue(CelPolicyElementId element, Severity severity, + absl::string_view message) { + issues_.push_back({element, severity, message}); +} + +void IssueReporter::ReportOffsetIssue(CelPolicyElementId element, + cel::SourcePosition relative_position, + Severity severity, + absl::string_view message) { + issues_.push_back({element, relative_position, severity, message}); +} + +void IssueReporter::ReportError(CelPolicyElementId element, + absl::string_view message) { + ReportIssue(element, Severity::kError, message); +} + +void IssueReporter::ReportError(CelPolicyElementId element, SourcePosition pos, + absl::string_view message) { + ReportOffsetIssue(element, pos, Severity::kError, message); +} + +} // namespace cel::policy_internal diff --git a/policy/internal/issue_reporter.h b/policy/internal/issue_reporter.h new file mode 100644 index 000000000..3f88806ef --- /dev/null +++ b/policy/internal/issue_reporter.h @@ -0,0 +1,57 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_ISSUE_REPORTER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_ISSUE_REPORTER_H_ + +#include + +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" + +namespace cel::policy_internal { + +class IssueReporter { + private: + using Severity = CelPolicyIssue::Severity; + + public: + void ReportIssue(CelPolicyElementId element, Severity severity, + absl::string_view message); + + void ReportOffsetIssue(CelPolicyElementId element, + cel::SourcePosition relative_position, + Severity severity, absl::string_view message); + + void ReportError(CelPolicyElementId element, absl::string_view message); + void ReportError(CelPolicyElementId element, SourcePosition relative_pos, + absl::string_view message); + + std::vector ReleaseIssues() { + using std::swap; + std::vector out; + swap(out, issues_); + return out; + } + const std::vector& issues() const { return issues_; } + + private: + std::vector issues_; +}; + +} // namespace cel::policy_internal + +#endif // THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_ISSUE_REPORTER_H_ diff --git a/policy/internal/optimizer_expr_factory.cc b/policy/internal/optimizer_expr_factory.cc new file mode 100644 index 000000000..6c89ae958 --- /dev/null +++ b/policy/internal/optimizer_expr_factory.cc @@ -0,0 +1,373 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/optimizer_expr_factory.h" + +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/container/flat_hash_map.h" +#include "absl/functional/any_invocable.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "common/ast.h" +#include "common/ast_rewrite.h" +#include "common/ast_traverse.h" +#include "common/ast_visitor_base.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/source.h" + +namespace cel { + +namespace { + +class MaxIdVisitor final : public AstVisitorBase { + public: + ExprId max_id() const { return max_id_; } + + void PreVisitExpr(const Expr& expr) override { + max_id_ = std::max(max_id_, expr.id()); + } + + void PostVisitExpr(const Expr&) override {} + + void PostVisitStruct(const Expr&, const StructExpr& struct_expr) override { + for (const auto& field : struct_expr.fields()) { + max_id_ = std::max(max_id_, field.id()); + } + } + + void PostVisitMap(const Expr&, const MapExpr& map_expr) override { + for (const auto& entry : map_expr.entries()) { + max_id_ = std::max(max_id_, entry.id()); + } + } + + private: + ExprId max_id_ = 0; +}; + +ExprId GetMaxId(const Expr& expr) { + MaxIdVisitor visitor; + AstTraverse(expr, visitor); + return visitor.max_id(); +} + +ExprId GetMaxId(const Ast& ast) { + ExprId max_id = GetMaxId(ast.root_expr()); + for (const auto& [id, _] : ast.source_info().positions()) { + max_id = std::max(max_id, id); + } + for (const auto& [id, expr] : ast.source_info().macro_calls()) { + max_id = std::max(max_id, id); + max_id = std::max(max_id, GetMaxId(expr)); + } + return max_id; +} + +// Replaces nested macros in a macro_calls expr with reference nodes. +// +// The macro_calls map is used for retaining the original structure of the +// parsed expression before macro expansion. When a macro appears inside another +// macro, the parser will replace the inner macro expr node with an unspecified +// expr with the inner macro's ID in the macro_calls map to save space. +class MakeMacroCallRewrite final : public AstRewriterBase { + public: + explicit MakeMacroCallRewrite(const SourceInfo& source_info) + : source_info_(source_info) {} + + bool PreVisitRewrite(Expr& expr) override { + if (source_info_.macro_calls().find(expr.id()) != + source_info_.macro_calls().end()) { + ExprId id = expr.id(); + expr.mutable_kind() = UnspecifiedExpr(); + expr.set_id(id); + return true; + } + return false; + } + + private: + const SourceInfo& source_info_; +}; + +// Updates macro_calls map entries to reflect a replaced expression in the +// main AST. +class ReplaceMacroCallRewrite final : public AstRewriterBase { + public: + ReplaceMacroCallRewrite(ExprId old_id, const Expr& replacement, + const SourceInfo& source_info) + : old_id_(old_id), replacement_(replacement), source_info_(source_info) {} + + bool PreVisitRewrite(Expr& expr) override { + if (expr.id() == old_id_) { + expr = macro_replacement(); + return true; + } + return false; + } + + Expr macro_replacement() { + if (!macro_replacement_) { + macro_replacement_.emplace(replacement_); + MakeMacroCallRewrite hole_creator(source_info_); + AstRewrite(*macro_replacement_, hole_creator); + } + return *macro_replacement_; + } + + private: + ExprId old_id_; + const Expr& replacement_; + absl::optional macro_replacement_; + const SourceInfo& source_info_; +}; + +void ReplaceSubExpr(Expr& expr, ExprId old_id, const Expr& replacement, + const SourceInfo& source_info) { + ReplaceMacroCallRewrite rewriter(old_id, replacement, source_info); + AstRewrite(expr, rewriter); +} + +class IdRewriter : public AstRewriterBase { + using CopyIdFn = absl::AnyInvocable; + + public: + explicit IdRewriter(CopyIdFn copy_id) : copy_id_(std::move(copy_id)) {} + + // No structure changes just ids. + bool PreVisitRewrite(Expr& expr) override { + expr.set_id(copy_id_(expr.id())); + if (expr.has_struct_expr()) { + for (auto& field : expr.mutable_struct_expr().mutable_fields()) { + field.set_id(copy_id_(field.id())); + } + } else if (expr.has_map_expr()) { + for (auto& entry : expr.mutable_map_expr().mutable_entries()) { + entry.set_id(copy_id_(entry.id())); + } + } + return false; + } + + private: + CopyIdFn copy_id_; +}; + +} // namespace + +OptimizerExprFactory::OptimizerExprFactory(Ast basis) + : ast_(std::move(basis)), next_id_(GetMaxId(ast_) + 1) {} + +OptimizerExprFactory::OptimizerExprFactory() : next_id_(1) {} + +Expr OptimizerExprFactory::Copy(const Expr& expr) { + Expr copied = expr; + IdRewriter rewriter([this](ExprId id) { return CopyId(id); }); + AstRewrite(copied, rewriter); + return copied; +} + +ListExprElement OptimizerExprFactory::Copy(const ListExprElement& element) { + return NewListElement(Copy(element.expr()), element.optional()); +} + +StructExprField OptimizerExprFactory::Copy(const StructExprField& field) { + auto field_id = CopyId(field.id()); + auto field_value = Copy(field.value()); + return NewStructField(field_id, field.name(), std::move(field_value), + field.optional()); +} + +MapExprEntry OptimizerExprFactory::Copy(const MapExprEntry& entry) { + auto entry_id = CopyId(entry.id()); + auto entry_key = Copy(entry.key()); + auto entry_value = Copy(entry.value()); + return NewMapEntry(entry_id, std::move(entry_key), std::move(entry_value), + entry.optional()); +} + +ExprId OptimizerExprFactory::NextId() { return next_id_++; } + +ExprId OptimizerExprFactory::CopyId(ExprId id) { + if (id == 0) { + return 0; + } + auto it = renumbers_.find(id); + if (it != renumbers_.end()) { + return it->second; + } + ExprId new_id = NextId(); + renumbers_[id] = new_id; + return new_id; +} + +SourceInfo OptimizerExprFactory::RemapSourceInfo(const SourceInfo& info, + SourcePosition offset) { + SourceInfo out; + + for (const auto& [old_id, macro_expr] : info.macro_calls()) { + if (auto it = renumbers_.find(old_id); it != renumbers_.end()) { + ExprId new_id = it->second; + out.mutable_macro_calls()[new_id] = Copy(macro_expr); + } + } + + for (const auto& [old_id, new_id] : renumbers_) { + if (auto it = info.positions().find(old_id); it != info.positions().end()) { + out.mutable_positions()[new_id] = it->second + offset; + } + } + + return out; +} + +void OptimizerExprFactory::MergeSourceInfo(const SourceInfo& info) { + auto& target_info = ast_.mutable_source_info(); + + for (const auto& [id, pos] : info.positions()) { + auto [it, inserted] = target_info.mutable_positions().insert({id, pos}); + if (!inserted) { + issues_.push_back(Issue{id, "conflicting ID in positions merge"}); + } + } + + for (const auto& [id, expr] : info.macro_calls()) { + auto [it, inserted] = target_info.mutable_macro_calls().insert({id, expr}); + if (!inserted) { + issues_.push_back(Issue{id, "conflicting ID in macro calls merge"}); + } + } + + // TODO(b/506179116): need to add some check that we aren't + // introducing incompatible tags. Not possible in the policy compiler right + // now. + for (const auto& ext : info.extensions()) { + auto& target_exts = target_info.mutable_extensions(); + if (!absl::c_linear_search(target_exts, ext)) { + target_exts.push_back(ext); + } + } +} + +void OptimizerExprFactory::RecordReplacement(ExprId id, const Expr& replacement, + bool keep_metadata) { + auto& source_info = ast_.mutable_source_info(); + if (!keep_metadata) { + source_info.mutable_positions().erase(id); + source_info.mutable_macro_calls().erase(id); + } + + for (auto& [macro_id, macro_expr] : source_info.mutable_macro_calls()) { + ReplaceSubExpr(macro_expr, id, replacement, source_info); + } +} + +Expr OptimizerExprFactory::ReportError(absl::string_view message) { + ExprId id = NextId(); + issues_.push_back(Issue{id, std::string(message)}); + return NewUnspecified(id); +} + +Expr OptimizerExprFactory::ReportErrorAt(const Expr& expr, + absl::string_view message) { + issues_.push_back(Issue{expr.id(), std::string(message)}); + return NewUnspecified(NextId()); +} + +Expr OptimizerExprFactory::ReportErrorAtCopy(const Expr& expr, + absl::string_view message) { + issues_.push_back(Issue{CopyId(expr.id()), std::string(message)}); + return NewUnspecified(NextId()); +} + +Expr OptimizerExprFactory::NewUnspecified() { return NewUnspecified(NextId()); } + +Expr OptimizerExprFactory::NewNullConst() { return NewNullConst(NextId()); } + +Expr OptimizerExprFactory::NewBoolConst(bool value) { + return NewBoolConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewIntConst(int64_t value) { + return NewIntConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewUintConst(uint64_t value) { + return NewUintConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewDoubleConst(double value) { + return NewDoubleConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewBytesConst(std::string value) { + return NewBytesConst(NextId(), std::move(value)); +} + +Expr OptimizerExprFactory::NewBytesConst(absl::string_view value) { + return NewBytesConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewBytesConst(const char* value) { + return NewBytesConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewStringConst(std::string value) { + return NewStringConst(NextId(), std::move(value)); +} + +Expr OptimizerExprFactory::NewStringConst(absl::string_view value) { + return NewStringConst(NextId(), value); +} + +Expr OptimizerExprFactory::NewStringConst(const char* value) { + return NewStringConst(NextId(), value); +} + +absl::flat_hash_map OptimizerExprFactory::ConsumeRenumbers() { + using std::swap; + absl::flat_hash_map out; + swap(out, renumbers_); + return out; +} + +void OptimizerExprFactory::StartCopyContext() { renumbers_.clear(); } + +const std::vector& OptimizerExprFactory::issues() + const { + return issues_; +} + +const Ast& OptimizerExprFactory::ast() const { return ast_; } + +Ast& OptimizerExprFactory::mutable_ast() { return ast_; } + +absl::string_view OptimizerExprFactory::AccuVarName() { + return ExprFactory::AccuVarName(); +} + +Expr OptimizerExprFactory::NewAccuIdent() { return NewAccuIdent(NextId()); } + +ExprId OptimizerExprFactory::CopyId(const Expr& expr) { + return CopyId(expr.id()); +} + +} // namespace cel diff --git a/policy/internal/optimizer_expr_factory.h b/policy/internal/optimizer_expr_factory.h new file mode 100644 index 000000000..6f63f1485 --- /dev/null +++ b/policy/internal/optimizer_expr_factory.h @@ -0,0 +1,419 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_OPTIMIZER_EXPR_FACTORY_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_OPTIMIZER_EXPR_FACTORY_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/source.h" + +namespace cel { + +class ParserMacroExprFactory; +class TestOptimizerExprFactory; + +// `OptimizerExprFactory` is a specialization of `ExprFactory` used for AST +// optimization. It provides utilities for correcting metadata for modified +// ASTs. +class OptimizerExprFactory : protected ExprFactory { + public: + struct Issue { + ExprId location = 0; + std::string message; + }; + + explicit OptimizerExprFactory(Ast basis); + OptimizerExprFactory(); + + protected: + using ExprFactory::IsArrayLike; + using ExprFactory::IsExprLike; + using ExprFactory::IsStringLike; + + template + struct IsRValue + : std::bool_constant< + std::disjunction_v, std::is_same>> {}; + + public: + // Consume the current set of renumberings. + absl::flat_hash_map ConsumeRenumbers(); + + // Starts a new copy context. The current set of renumberings are cleared. + void StartCopyContext(); + + const std::vector& issues() const; + + // Record that a node in the working AST was replaced. This is used to correct + // metadata referencing the old ID. + void RecordReplacement(ExprId id, const Expr& replacement, + bool keep_metadata = false); + + // Makes a copy of source metadata that is remapped to new expr Ids using + // current renumberings. This is suitable for merging into the main source + // info. + SourceInfo RemapSourceInfo(const SourceInfo& info, SourcePosition offset = 0); + + // Merge a remapped SourceInfo into the current one. + void MergeSourceInfo(const SourceInfo& info); + + const Ast& ast() const; + Ast& mutable_ast(); + + absl::string_view AccuVarName(); + + ABSL_MUST_USE_RESULT Expr Copy(const Expr& expr); + + ABSL_MUST_USE_RESULT ListExprElement Copy(const ListExprElement& element); + + ABSL_MUST_USE_RESULT StructExprField Copy(const StructExprField& field); + + ABSL_MUST_USE_RESULT MapExprEntry Copy(const MapExprEntry& entry); + + ABSL_MUST_USE_RESULT Expr NewUnspecified(); + + ABSL_MUST_USE_RESULT Expr NewNullConst(); + + ABSL_MUST_USE_RESULT Expr NewBoolConst(bool value); + + ABSL_MUST_USE_RESULT Expr NewIntConst(int64_t value); + + ABSL_MUST_USE_RESULT Expr NewUintConst(uint64_t value); + + ABSL_MUST_USE_RESULT Expr NewDoubleConst(double value); + + ABSL_MUST_USE_RESULT Expr NewBytesConst(std::string value); + + ABSL_MUST_USE_RESULT Expr NewBytesConst(absl::string_view value); + + ABSL_MUST_USE_RESULT Expr NewBytesConst(const char* absl_nullable value); + + ABSL_MUST_USE_RESULT Expr NewStringConst(std::string value); + + ABSL_MUST_USE_RESULT Expr NewStringConst(absl::string_view value); + + ABSL_MUST_USE_RESULT Expr NewStringConst(const char* absl_nullable value); + + template ::value>> + ABSL_MUST_USE_RESULT Expr NewIdent(Name name); + + ABSL_MUST_USE_RESULT Expr NewAccuIdent(); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewSelect(Operand operand, Field field); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewPresenceTest(Operand operand, Field field); + + template < + typename Function, typename... Args, + typename = std::enable_if_t::value>, + typename = std::enable_if_t...>>> + ABSL_MUST_USE_RESULT Expr NewCall(Function function, Args&&... args); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewCall(Function function, Args args); + + template < + typename Function, typename Target, typename... Args, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t...>>> + ABSL_MUST_USE_RESULT Expr NewMemberCall(Function function, Target target, + Args&&... args); + + template ::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewMemberCall(Function function, Target target, + Args args); + + using ExprFactory::NewListElement; + + template ...>>> + ABSL_MUST_USE_RESULT Expr NewList(Elements&&... elements); + + template ::value>> + ABSL_MUST_USE_RESULT Expr NewList(Elements elements); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT StructExprField NewStructField(Name name, Value value, + bool optional = false); + + template ::value>, + typename = std::enable_if_t< + std::conjunction_v...>>> + ABSL_MUST_USE_RESULT Expr NewStruct(Name name, Fields&&... fields); + + template < + typename Name, typename Fields, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewStruct(Name name, Fields fields); + + template ::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT MapExprEntry NewMapEntry(Key key, Value value, + bool optional = false); + + template ...>>> + ABSL_MUST_USE_RESULT Expr NewMap(Entries&&... entries); + + template ::value>> + ABSL_MUST_USE_RESULT Expr NewMap(Entries entries); + + template ::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewComprehension(IterVar iter_var, + IterRange iter_range, + AccuVar accu_var, + AccuInit accu_init, + LoopCondition loop_condition, + LoopStep loop_step, Result result); + + template ::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>, + typename = std::enable_if_t::value>> + ABSL_MUST_USE_RESULT Expr NewComprehension( + IterVar iter_var, IterVar2 iter_var2, IterRange iter_range, + AccuVar accu_var, AccuInit accu_init, LoopCondition loop_condition, + LoopStep loop_step, Result result); + + ABSL_MUST_USE_RESULT Expr ReportError(absl::string_view message); + + // Reports an error at the id in the optimized AST. + ABSL_MUST_USE_RESULT Expr ReportErrorAt(const Expr& expr, + absl::string_view message); + // Reports an error at the mapped id of the copy of expr in the optimized AST. + ABSL_MUST_USE_RESULT Expr ReportErrorAtCopy(const Expr& expr, + absl::string_view message); + + protected: + ABSL_MUST_USE_RESULT ExprId NextId(); + + ABSL_MUST_USE_RESULT ExprId CopyId(ExprId id); + + ABSL_MUST_USE_RESULT ExprId CopyId(const Expr& expr); + + using ExprFactory::AccuVarName; + using ExprFactory::NewAccuIdent; + using ExprFactory::NewBoolConst; + using ExprFactory::NewBytesConst; + using ExprFactory::NewCall; + using ExprFactory::NewComprehension; + using ExprFactory::NewConst; + using ExprFactory::NewDoubleConst; + using ExprFactory::NewIdent; + using ExprFactory::NewIntConst; + using ExprFactory::NewList; + using ExprFactory::NewMap; + using ExprFactory::NewMapEntry; + using ExprFactory::NewMemberCall; + using ExprFactory::NewNullConst; + using ExprFactory::NewPresenceTest; + using ExprFactory::NewSelect; + using ExprFactory::NewStringConst; + using ExprFactory::NewStruct; + using ExprFactory::NewStructField; + using ExprFactory::NewUintConst; + using ExprFactory::NewUnspecified; + + private: + Ast ast_; + absl::flat_hash_map renumbers_; + std::vector issues_; + + ExprId next_id_ = 1; +}; + +// Implementation details. + +template +Expr OptimizerExprFactory::NewIdent(Name name) { + return NewIdent(NextId(), std::move(name)); +} + +template +Expr OptimizerExprFactory::NewSelect(Operand operand, Field field) { + return NewSelect(NextId(), std::move(operand), std::move(field)); +} + +template +Expr OptimizerExprFactory::NewPresenceTest(Operand operand, Field field) { + return NewPresenceTest(NextId(), std::move(operand), std::move(field)); +} + +template +Expr OptimizerExprFactory::NewCall(Function function, Args&&... args) { + std::vector array; + array.reserve(sizeof...(Args)); + (array.push_back(std::forward(args)), ...); + return NewCall(NextId(), std::move(function), std::move(array)); +} + +template +Expr OptimizerExprFactory::NewCall(Function function, Args args) { + return NewCall(NextId(), std::move(function), std::move(args)); +} + +template +Expr OptimizerExprFactory::NewMemberCall(Function function, Target target, + Args&&... args) { + std::vector array; + array.reserve(sizeof...(Args)); + (array.push_back(std::forward(args)), ...); + return NewMemberCall(NextId(), std::move(function), std::move(target), + std::move(array)); +} + +template +Expr OptimizerExprFactory::NewMemberCall(Function function, Target target, + Args args) { + return NewMemberCall(NextId(), std::move(function), std::move(target), + std::move(args)); +} + +template +Expr OptimizerExprFactory::NewList(Elements&&... elements) { + std::vector array; + array.reserve(sizeof...(Elements)); + (array.push_back(std::forward(elements)), ...); + return NewList(NextId(), std::move(array)); +} + +template +Expr OptimizerExprFactory::NewList(Elements elements) { + return NewList(NextId(), std::move(elements)); +} + +template +StructExprField OptimizerExprFactory::NewStructField(Name name, Value value, + bool optional) { + return NewStructField(NextId(), std::move(name), std::move(value), optional); +} + +template +Expr OptimizerExprFactory::NewStruct(Name name, Fields&&... fields) { + std::vector array; + array.reserve(sizeof...(Fields)); + (array.push_back(std::forward(fields)), ...); + return NewStruct(NextId(), std::move(name), std::move(array)); +} + +template +Expr OptimizerExprFactory::NewStruct(Name name, Fields fields) { + return NewStruct(NextId(), std::move(name), std::move(fields)); +} + +template +MapExprEntry OptimizerExprFactory::NewMapEntry(Key key, Value value, + bool optional) { + return NewMapEntry(NextId(), std::move(key), std::move(value), optional); +} + +template +Expr OptimizerExprFactory::NewMap(Entries&&... entries) { + std::vector array; + array.reserve(sizeof...(Entries)); + (array.push_back(std::forward(entries)), ...); + return NewMap(NextId(), std::move(array)); +} + +template +Expr OptimizerExprFactory::NewMap(Entries entries) { + return NewMap(NextId(), std::move(entries)); +} + +template +Expr OptimizerExprFactory::NewComprehension(IterVar iter_var, + IterRange iter_range, + AccuVar accu_var, + AccuInit accu_init, + LoopCondition loop_condition, + LoopStep loop_step, Result result) { + return NewComprehension(NextId(), std::move(iter_var), std::move(iter_range), + std::move(accu_var), std::move(accu_init), + std::move(loop_condition), std::move(loop_step), + std::move(result)); +} + +template +Expr OptimizerExprFactory::NewComprehension( + IterVar iter_var, IterVar2 iter_var2, IterRange iter_range, + AccuVar accu_var, AccuInit accu_init, LoopCondition loop_condition, + LoopStep loop_step, Result result) { + return NewComprehension(NextId(), std::move(iter_var), std::move(iter_var2), + std::move(iter_range), std::move(accu_var), + std::move(accu_init), std::move(loop_condition), + std::move(loop_step), std::move(result)); +} + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_OPTIMIZER_EXPR_FACTORY_H_ diff --git a/policy/internal/optimizer_expr_factory_test.cc b/policy/internal/optimizer_expr_factory_test.cc new file mode 100644 index 000000000..1b14b5628 --- /dev/null +++ b/policy/internal/optimizer_expr_factory_test.cc @@ -0,0 +1,570 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/optimizer_expr_factory.h" + +#include +#include +#include + +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/ast_proto.h" +#include "common/ast_rewrite.h" +#include "common/decl.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/source.h" +#include "common/type.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "testutil/expr_printer.h" +#include "tools/cel_unparser.h" + +namespace cel { + +using ::testing::SizeIs; + +// Expose protected members of OptimizerExprFactory for use in tests +// +// These allow setting explicit IDs which is not safe for the optimizing +// factory. +class TestOptimizerExprFactory final : public OptimizerExprFactory { + public: + using OptimizerExprFactory::OptimizerExprFactory; + + using OptimizerExprFactory::NewBoolConst; + using OptimizerExprFactory::NewCall; + using OptimizerExprFactory::NewComprehension; + using OptimizerExprFactory::NewIdent; + using OptimizerExprFactory::NewList; + using OptimizerExprFactory::NewListElement; + using OptimizerExprFactory::NewMap; + using OptimizerExprFactory::NewMapEntry; + using OptimizerExprFactory::NewMemberCall; + using OptimizerExprFactory::NewSelect; + using OptimizerExprFactory::NewStruct; + using OptimizerExprFactory::NewStructField; + using OptimizerExprFactory::NewUnspecified; + using OptimizerExprFactory::NextId; +}; + +namespace { + +class ReplaceExprRewriter final : public AstRewriterBase { + public: + ReplaceExprRewriter(ExprId old_id, const Expr& replacement) + : old_id_(old_id), replacement_(replacement) {} + + bool PreVisitRewrite(Expr& expr) override { + if (expr.id() == old_id_) { + expr = replacement_; + return true; + } + return false; + } + + private: + ExprId old_id_; + const Expr& replacement_; +}; + +void ReplaceExprInTree(Expr& expr, ExprId old_id, const Expr& replacement) { + ReplaceExprRewriter rewriter(old_id, replacement); + AstRewrite(expr, rewriter); +} + +absl::StatusOr> CreateTestCompiler() { + CompilerOptions opts; + opts.parser_options.add_macro_calls = true; + CEL_ASSIGN_OR_RETURN( + auto builder, cel::NewCompilerBuilder( + cel::internal::GetSharedTestingDescriptorPool(), opts)); + CEL_RETURN_IF_ERROR(builder->AddLibrary(cel::StandardCompilerLibrary())); + CEL_RETURN_IF_ERROR(builder->GetCheckerBuilder().AddVariable( + cel::MakeVariableDecl("to_replace", cel::DynType()))); + return builder->Build(); +} + +TEST(OptimizerExprFactory, CopyUnspecified) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ(factory.Copy(factory.NewUnspecified()), factory.NewUnspecified(2)); +} + +TEST(OptimizerExprFactory, CopyIdent) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ(factory.Copy(factory.NewIdent("foo")), factory.NewIdent(2, "foo")); +} + +TEST(OptimizerExprFactory, CopyConst) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ(factory.Copy(factory.NewBoolConst(true)), + factory.NewBoolConst(2, true)); +} + +TEST(OptimizerExprFactory, CopySelect) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ(factory.Copy(factory.NewSelect(factory.NewIdent("foo"), "bar")), + factory.NewSelect(3, factory.NewIdent(4, "foo"), "bar")); +} + +TEST(OptimizerExprFactory, CopyCall) { + TestOptimizerExprFactory factory{Ast()}; + std::vector copied_args; + copied_args.reserve(1); + copied_args.push_back(factory.NewIdent(6, "baz")); + EXPECT_EQ(factory.Copy(factory.NewMemberCall("bar", factory.NewIdent("foo"), + factory.NewIdent("baz"))), + factory.NewMemberCall(4, "bar", factory.NewIdent(5, "foo"), + absl::MakeSpan(copied_args))); +} + +TEST(OptimizerExprFactory, CopyList) { + TestOptimizerExprFactory factory{Ast()}; + std::vector copied_elements; + copied_elements.reserve(1); + copied_elements.push_back(factory.NewListElement(factory.NewIdent(4, "foo"))); + EXPECT_EQ(factory.Copy(factory.NewList( + factory.NewListElement(factory.NewIdent("foo")))), + factory.NewList(3, absl::MakeSpan(copied_elements))); +} + +TEST(OptimizerExprFactory, CopyStruct) { + TestOptimizerExprFactory factory{Ast()}; + std::vector copied_fields; + copied_fields.reserve(1); + copied_fields.push_back( + factory.NewStructField(5, "bar", factory.NewIdent(6, "baz"))); + EXPECT_EQ(factory.Copy(factory.NewStruct( + "foo", factory.NewStructField("bar", factory.NewIdent("baz")))), + factory.NewStruct(4, "foo", absl::MakeSpan(copied_fields))); +} + +TEST(OptimizerExprFactory, CopyMap) { + TestOptimizerExprFactory factory{Ast()}; + std::vector copied_entries; + copied_entries.reserve(1); + copied_entries.push_back(factory.NewMapEntry(6, factory.NewIdent(7, "bar"), + factory.NewIdent(8, "baz"))); + EXPECT_EQ(factory.Copy(factory.NewMap(factory.NewMapEntry( + factory.NewIdent("bar"), factory.NewIdent("baz")))), + factory.NewMap(5, absl::MakeSpan(copied_entries))); +} + +TEST(OptimizerExprFactory, CopyComprehension) { + TestOptimizerExprFactory factory{Ast()}; + EXPECT_EQ( + factory.Copy(factory.NewComprehension( + "foo", factory.NewList(), "bar", factory.NewBoolConst(true), + factory.NewIdent("baz"), factory.NewIdent("foo"), + factory.NewIdent("bar"))), + factory.NewComprehension( + 7, "foo", factory.NewList(8, std::vector()), "bar", + factory.NewBoolConst(9, true), factory.NewIdent(10, "baz"), + factory.NewIdent(11, "foo"), factory.NewIdent(12, "bar"))); +} + +TEST(OptimizerExprFactory, RemapSourceInfo) { + TestOptimizerExprFactory factory{Ast()}; + Expr orig = factory.NewIdent("foo"); // allocates ID 1 + Expr copied = factory.Copy(orig); // copies ID 1 to mapped ID 2 + + SourceInfo info; + info.mutable_positions()[1] = 42; // old ID 1 has position 42 + + SourceInfo remapped = factory.RemapSourceInfo(info, 10); + + // remapped should have ID 2 mapped to position 42 + 10 = 52 + auto it = remapped.positions().find(2); + ASSERT_NE(it, remapped.positions().end()); + EXPECT_EQ(it->second, 52); +} + +TEST(OptimizerExprFactory, RemapSourceInfoWithMacroCalls) { + TestOptimizerExprFactory factory{Ast()}; + Expr orig = factory.NewIdent("foo"); // allocates ID 1 + Expr copied = factory.Copy(orig); // copies ID 1 to mapped ID 2 + + SourceInfo info; + // old ID 1 has macro call with ID 3 + info.mutable_macro_calls()[1] = factory.NewIdent("bar"); + + SourceInfo remapped = factory.RemapSourceInfo(info, 10); + + // remapped should have ID 2 mapped to the copied macro call + // since "bar" has ID 3, Copy(bar) should map ID 3 to ID 4 + + auto it = remapped.macro_calls().find(2); + ASSERT_NE(it, remapped.macro_calls().end()); + + // The macro call should be an Ident with new ID 4 + EXPECT_EQ(it->second.id(), 4); + EXPECT_TRUE(it->second.has_ident_expr()); + EXPECT_EQ(it->second.ident_expr().name(), "bar"); +} + +TEST(OptimizerExprFactory, ReportError) { + TestOptimizerExprFactory factory{Ast()}; + Expr err_expr = factory.ReportError("something went wrong"); + + // err_expr should be unspecified with ID 1 + EXPECT_EQ(err_expr.id(), 1); + EXPECT_EQ(err_expr.kind_case(), ExprKindCase::kUnspecifiedExpr); + + // issues_ should have 1 entry with ID 1 and correct message + ASSERT_EQ(factory.issues().size(), 1); + EXPECT_EQ(factory.issues()[0].location, 1); + EXPECT_EQ(factory.issues()[0].message, "something went wrong"); +} + +TEST(OptimizerExprFactory, ReportErrorAt) { + TestOptimizerExprFactory factory{Ast()}; + Expr orig = factory.NewIdent("foo"); // allocates ID 1 + Expr copied = factory.Copy(orig); // copies ID 1 to mapped ID 2 + + Expr err_expr = factory.ReportErrorAtCopy(orig, "error on foo"); + + // err_expr should be unspecified with ID 3 (NextId) + EXPECT_EQ(err_expr.id(), 3); + EXPECT_EQ(err_expr.kind_case(), ExprKindCase::kUnspecifiedExpr); + + // issues_ should have 1 entry with mapped ID 2 and correct message + ASSERT_EQ(factory.issues().size(), 1); + EXPECT_EQ(factory.issues()[0].location, 2); + EXPECT_EQ(factory.issues()[0].message, "error on foo"); +} + +TEST(OptimizerExprFactory, MergeSourceInfo) { + // Create a base AST with some source info + SourceInfo base_info; + base_info.set_syntax_version("cel1"); + base_info.set_location("test.cel"); + base_info.mutable_positions()[1] = 10; + + Ast base_ast(Expr(), std::move(base_info)); + + TestOptimizerExprFactory factory{std::move(base_ast)}; + + // Create a new source info to merge + SourceInfo new_info; + new_info.mutable_positions()[2] = 20; + + factory.MergeSourceInfo(new_info); + + // The merged source info should have both positions + const auto& merged_info = factory.ast().source_info(); + EXPECT_EQ(merged_info.syntax_version(), "cel1"); + EXPECT_EQ(merged_info.location(), "test.cel"); + + auto it1 = merged_info.positions().find(1); + ASSERT_NE(it1, merged_info.positions().end()); + EXPECT_EQ(it1->second, 10); + + auto it2 = merged_info.positions().find(2); + ASSERT_NE(it2, merged_info.positions().end()); + EXPECT_EQ(it2->second, 20); +} + +TEST(OptimizerExprFactory, MergeSourceInfoConflict) { + SourceInfo base_info; + base_info.mutable_positions()[1] = 10; + + Ast base_ast(Expr(), std::move(base_info)); + TestOptimizerExprFactory factory{std::move(base_ast)}; + + SourceInfo new_info; + new_info.mutable_positions()[1] = 20; // conflicting ID 1 + + factory.MergeSourceInfo(new_info); + + // Should report an error for the conflict + ASSERT_EQ(factory.issues().size(), 1); + EXPECT_EQ(factory.issues()[0].location, 1); + EXPECT_EQ(factory.issues()[0].message, "conflicting ID in positions merge"); +} + +TEST(OptimizerExprFactory, RecordReplacement) { + SourceInfo base_info; + base_info.mutable_positions()[1] = 10; + base_info.mutable_positions()[2] = 20; + + TestOptimizerExprFactory factory{Ast()}; + + // macro_calls[1] maps ID 1 to macro call "bar(foo)" (where "foo" has ID 1) + base_info.mutable_macro_calls()[1] = + factory.NewCall("bar", factory.NewIdent(1, "foo")); + + // macro_calls[2] maps ID 2 to macro call "baz(foo)" (where "foo" has ID 1) + base_info.mutable_macro_calls()[2] = + factory.NewCall("baz", factory.NewIdent(1, "foo")); + + Ast base_ast(Expr(), std::move(base_info)); + TestOptimizerExprFactory optimizer{std::move(base_ast)}; + + // Record the replacement of ID 1 by a new Ident "replacement" with ID 3 + optimizer.RecordReplacement(1, factory.NewIdent(3, "replacement")); + + const auto& result_info = optimizer.ast().source_info(); + + // 1. ID 1 should be erased from positions + EXPECT_EQ(result_info.positions().find(1), result_info.positions().end()); + EXPECT_NE(result_info.positions().find(2), result_info.positions().end()); + + // 2. ID 1 should be erased from macro_calls keys + EXPECT_EQ(result_info.macro_calls().find(1), result_info.macro_calls().end()); + + // 3. macro_calls[2] should still exist, but its argument referencing ID 1 + // should be replaced with the Ident "replacement" with ID 3 inline + auto it = result_info.macro_calls().find(2); + ASSERT_NE(it, result_info.macro_calls().end()); + + const Expr& macro_expr = it->second; + ASSERT_TRUE(macro_expr.has_call_expr()); + ASSERT_EQ(macro_expr.call_expr().args().size(), 1); + + const Expr& arg = macro_expr.call_expr().args()[0]; + EXPECT_EQ(arg.id(), 3); + EXPECT_TRUE(arg.has_ident_expr()); + EXPECT_EQ(arg.ident_expr().name(), "replacement"); +} + +class IdAdorner : public cel::test::ExpressionAdorner { + public: + std::string Adorn(const cel::Expr& e) const override { + return absl::StrCat("#", e.id()); + } + + std::string AdornStructField(const cel::StructExprField& e) const override { + return absl::StrCat("#", e.id()); + } + + std::string AdornMapEntry(const cel::MapExprEntry& e) const override { + return absl::StrCat("#", e.id()); + } +}; + +TEST(OptimizerExprFactory, UnparseCopiedMacroCall) { + // Arrange: create an template expression and one to inline. + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateTestCompiler()); + + ASSERT_OK_AND_ASSIGN(auto basis_result, + compiler->Compile("[1].map(x, x + to_replace)")); + ASSERT_TRUE(basis_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto basis_ast, basis_result.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN(auto copy_result, + compiler->Compile("[1].filter(x, x > 2).size()")); + ASSERT_TRUE(copy_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto copy_ast, copy_result.ReleaseAst()); + + // Locate the "to_replace" IdentExpr node in reference_map + ExprId to_replace_id = 0; + for (const auto& [id, ref] : basis_ast->reference_map()) { + if (ref.name() == "to_replace") { + to_replace_id = id; + break; + } + } + ASSERT_NE(to_replace_id, 0); + + // Act: implement the optimization. + TestOptimizerExprFactory factory{std::move(*basis_ast)}; + Expr copied_expr = factory.Copy(copy_ast->root_expr()); + SourceInfo remapped_info = factory.RemapSourceInfo(copy_ast->source_info()); + factory.MergeSourceInfo(remapped_info); + + ReplaceExprInTree(factory.mutable_ast().mutable_root_expr(), to_replace_id, + copied_expr); + factory.RecordReplacement(to_replace_id, copied_expr); + + // Test AST structure. + EXPECT_EQ( + cel::test::ExprPrinter(IdAdorner()).Print(factory.ast().root_expr()), + R"(__comprehension__( + // Variable + x, + // Target + [ + 1#2 + ]#1, + // Accumulator + @result, + // Init + []#8, + // LoopCondition + true#9, + // LoopStep + _+_( + @result#10, + [ + _+_( + x#5, + __comprehension__( + // Variable + x, + // Target + [ + 1#18 + ]#17, + // Accumulator + @result, + // Init + []#19, + // LoopCondition + true#20, + // LoopStep + _?_:_( + _>_( + x#23, + 2#24 + )#22, + _+_( + @result#26, + [ + x#28 + ]#27 + )#25, + @result#29 + )#21, + // Result + @result#30)#16.size()#15 + )#6 + ]#11 + )#12, + // Result + @result#13)#14)"); + + // Check that the structure is compatible with unparser. + cel::expr::ParsedExpr optimized_parsed; + auto status = AstToParsedExpr(factory.ast(), &optimized_parsed); + ASSERT_THAT(status, absl_testing::IsOk()); + ASSERT_OK_AND_ASSIGN(std::string unparsed, + google::api::expr::Unparse(optimized_parsed)); + + EXPECT_EQ(unparsed, "[1].map(x, x + [1].filter(x, x > 2).size())"); + + const CallExpr& call_expr = factory.mutable_ast() + .mutable_source_info() + .mutable_macro_calls()[14] + .mutable_call_expr(); + ASSERT_THAT(call_expr.args(), SizeIs(2)); + ASSERT_THAT(call_expr.args()[1].call_expr().args(), SizeIs(2)); + EXPECT_EQ(call_expr.args()[1].call_expr().args()[1].id(), 15); + + EXPECT_EQ(call_expr.args()[1].call_expr().args()[1].call_expr().target().id(), + 16); + EXPECT_EQ(call_expr.args()[1] + .call_expr() + .args()[1] + .call_expr() + .target() + .kind_case(), + ExprKindCase::kUnspecifiedExpr); +} + +TEST(OptimizerExprFactory, CopyMultipleAstsWithConsumeRenumbers) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + CreateTestCompiler()); + + ASSERT_OK_AND_ASSIGN(auto ast1_result, compiler->Compile("[1]")); + ASSERT_TRUE(ast1_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast1, ast1_result.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN(auto ast2_result, compiler->Compile("2")); + ASSERT_TRUE(ast2_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast2, ast2_result.ReleaseAst()); + + TestOptimizerExprFactory factory{Ast()}; + + Expr copied1 = factory.Copy(ast1->root_expr()); + auto renumbers1 = factory.ConsumeRenumbers(); + + Expr copied2 = factory.Copy(ast2->root_expr()); + auto renumbers2 = factory.ConsumeRenumbers(); + + EXPECT_EQ(renumbers1.size(), 2); + EXPECT_EQ(renumbers2.size(), 1); + + EXPECT_NE(copied1.id(), copied2.id()); + EXPECT_GT(copied2.id(), copied1.id()); +} + +TEST(OptimizerExprFactory, MaxIdVisitorExprKinds) { + ASSERT_OK_AND_ASSIGN(auto compiler, CreateTestCompiler()); + + // Expression that covers all the kinds. + ASSERT_OK_AND_ASSIGN(auto source, NewSource(R"cel( + Struct{field : 1} || + {'key' : 'value'} || [1].exists(x, x) || foo(bar))cel")); + ASSERT_OK_AND_ASSIGN(auto ast, compiler->GetParser().Parse(*source)); + + TestOptimizerExprFactory factory{std::move(*ast)}; + + EXPECT_EQ(factory.NextId(), 26); +} + +TEST(OptimizerExprFactory, CopyListElement) { + TestOptimizerExprFactory factory{Ast()}; + ListExprElement orig = factory.NewListElement(factory.NewIdent("foo")); + ListExprElement copied = factory.Copy(orig); + EXPECT_EQ(copied.expr(), factory.NewIdent(2, "foo")); +} + +TEST(OptimizerExprFactory, CopyStructField) { + TestOptimizerExprFactory factory{Ast()}; + StructExprField orig = factory.NewStructField("bar", factory.NewIdent("baz")); + StructExprField copied = factory.Copy(orig); + EXPECT_EQ(copied.id(), 3); + EXPECT_EQ(copied.name(), "bar"); + EXPECT_EQ(copied.value(), factory.NewIdent(4, "baz")); +} + +TEST(OptimizerExprFactory, CopyMapEntry) { + TestOptimizerExprFactory factory{Ast()}; + MapExprEntry orig = + factory.NewMapEntry(factory.NewIdent("bar"), factory.NewIdent("baz")); + MapExprEntry copied = factory.Copy(orig); + EXPECT_EQ(copied.id(), 4); + EXPECT_EQ(copied.key(), factory.NewIdent(5, "bar")); + EXPECT_EQ(copied.value(), factory.NewIdent(6, "baz")); +} + +TEST(OptimizerExprFactory, MergeSourceInfoMacroConflict) { + SourceInfo base_info; + base_info.mutable_macro_calls()[1] = Expr(); + + Ast base_ast(Expr(), std::move(base_info)); + TestOptimizerExprFactory factory{std::move(base_ast)}; + + SourceInfo new_info; + new_info.mutable_macro_calls()[1] = Expr(); + + factory.MergeSourceInfo(new_info); + + ASSERT_EQ(factory.issues().size(), 1); + EXPECT_EQ(factory.issues()[0].location, 1); + EXPECT_EQ(factory.issues()[0].message, "conflicting ID in macro calls merge"); +} + +} // namespace +} // namespace cel diff --git a/policy/test_custom_yaml_policy_parser.cc b/policy/test_custom_yaml_policy_parser.cc new file mode 100644 index 000000000..faced6952 --- /dev/null +++ b/policy/test_custom_yaml_policy_parser.cc @@ -0,0 +1,188 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "internal/status_macros.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_context.h" +#include "policy/cel_policy_parser.h" +#include "policy/yaml_policy_parser.h" +#include "yaml-cpp/node/node.h" +#include "yaml-cpp/yaml.h" // IWYU pragma: keep + +namespace cel::internal { + +// TestCustomYamlPolicyParser is used to support unit tests for custom tags +// and custom policy structures. It demonstrates the versatility of the +// cel::YamlPolicyParser framework API by implementing custom tag and block +// parsing without needing to modify the core parser. +class TestCustomYamlPolicyParser : public cel::YamlPolicyParser { + absl::StatusOr ParsePolicyTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node) const override { + if (tag_name.value() == "name" || tag_name.value() == "description" || + tag_name.value() == "imports") { + return cel::YamlPolicyParser::ParsePolicyTag(ctx, tag_name, node); + } + if (tag_name.value() == "purpose") { + std::optional purpose = + GetValueString(ctx, node, "Policy purpose is not a string"); + if (purpose.has_value()) { + ctx.policy().mutable_metadata()["purpose"] = *purpose; + } + return true; + } + if (tag_name.value() == "version") { + std::optional version = + GetValueString(ctx, node, "Policy version is not a string"); + if (!version.has_value()) { + return true; + } + int version_int; + if (!absl::SimpleAtoi(version->value(), &version_int)) { + ctx.ReportError(version->id(), + absl::StrCat("Policy version is not an integer: ", + version->value())); + return true; + } + ctx.policy().mutable_metadata()["version"] = version_int; + return true; + } + + if (tag_name.value() == "conditions") { + if (!node.IsSequence()) { + ctx.ReportError(tag_name.id(), "Policy 'conditions' is not a sequence"); + return true; + } + for (const YAML::Node& condition : node) { + // Track the number of existing matches before parsing. When ParseMatch + // evaluates an 'else' block, it recursively triggers parsing and adds + // internal inner matches directly to the rule's match vector. + // Inserting the outer match at begin() + size_before ensures that the + // primary outer 'if' condition is always evaluated before its nested + // 'else' fallbacks. + // + // Example: + // if: x > 0 + // then: "positive" + // else: "negative" + // + // The inner "negative" match is parsed and appended to rule.matches() + // by the inner recursive call, before the outer "x > 0" match finishes. + // Inserting at size_before places the "x > 0" match ahead of the inner + // one. + size_t size_before = ctx.policy().rule().matches().size(); + CEL_ASSIGN_OR_RETURN(Match match, + cel::YamlPolicyParser::ParseMatch( + ctx, condition, ctx.policy().mutable_rule())); + ctx.policy().mutable_rule().mutable_matches().insert( + ctx.policy().mutable_rule().mutable_matches().begin() + size_before, + std::move(match)); + } + + return true; + } + return false; + } + + absl::Status ParseThenBlock(CelPolicyParseContext& ctx, + const YAML::Node& value_node, + Match& match) const { + if (value_node.IsScalar()) { + std::optional val = GetValueString( + ctx, value_node, "Policy condition 'then' is not a string"); + if (val.has_value()) { + OutputBlock output; + output.set_output(*val); + match.set_result(output); + } + } else if (value_node.IsMap()) { + auto nested_rule = std::make_unique(); + CEL_ASSIGN_OR_RETURN( + Match nested_match, + cel::YamlPolicyParser::ParseMatch(ctx, value_node, *nested_rule)); + nested_rule->mutable_matches().insert( + nested_rule->mutable_matches().begin(), std::move(nested_match)); + match.set_result(std::move(nested_rule)); + } else { + ctx.ReportError(CollectMetadata(ctx, value_node), + "Bad syntax in 'if/then' block"); + } + return absl::OkStatus(); + } + + absl::Status ParseElseBlock(CelPolicyParseContext& ctx, + const YAML::Node& value_node, Rule& rule) const { + if (value_node.IsScalar()) { + std::optional val = GetValueString( + ctx, value_node, "Policy condition 'else' is not a string"); + if (val.has_value()) { + Match else_match; + else_match.set_id(CollectMetadata(ctx, value_node)); + OutputBlock output; + output.set_output(*val); + else_match.set_result(output); + rule.mutable_matches().push_back(std::move(else_match)); + } + } else if (value_node.IsMap()) { + size_t size_before = rule.matches().size(); + CEL_ASSIGN_OR_RETURN(Match match, cel::YamlPolicyParser::ParseMatch( + ctx, value_node, rule)); + rule.mutable_matches().insert( + rule.mutable_matches().begin() + size_before, std::move(match)); + } else { + ctx.ReportError(CollectMetadata(ctx, value_node), + "Bad syntax in 'if/then' block"); + } + return absl::OkStatus(); + } + + absl::StatusOr ParseMatchTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, Match& match, + Rule& rule) const override { + if (tag_name.value() == "if") { + std::optional condition = + GetValueString(ctx, node, "Policy 'if' condition is not a string"); + if (condition.has_value()) { + match.set_condition(*condition); + } + return true; + } + if (tag_name.value() == "then") { + CEL_RETURN_IF_ERROR(ParseThenBlock(ctx, node, match)); + return true; + } + if (tag_name.value() == "else") { + CEL_RETURN_IF_ERROR(ParseElseBlock(ctx, node, rule)); + return true; + } + return false; + } +}; + +const CelPolicyParser& GetTestCustomYamlPolicyParser() { + static const auto* const parser = new TestCustomYamlPolicyParser(); + return *parser; +} + +} // namespace cel::internal diff --git a/policy/test_util.cc b/policy/test_util.cc new file mode 100644 index 000000000..9fe1e43d1 --- /dev/null +++ b/policy/test_util.cc @@ -0,0 +1,221 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +#include "policy/test_util.h" + +#include +#include +#include +#include + +#include "cel/expr/eval.pb.h" +#include "cel/expr/value.pb.h" +#include "google/protobuf/struct.pb.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "internal/status_macros.h" +#include "yaml-cpp/yaml.h" + +namespace cel::test { + +namespace { + +absl::Status YamlToExprValue(const YAML::Node& node, + cel::expr::Value* proto) { + if (node.IsNull()) { + proto->set_null_value(google::protobuf::NULL_VALUE); + return absl::OkStatus(); + } + if (node.IsScalar()) { + // Try bool + try { + proto->set_bool_value(node.as()); + return absl::OkStatus(); + } catch (...) { + } + // Try int64 + try { + int64_t val; + if (YAML::convert::decode(node, val)) { + proto->set_int64_value(val); + return absl::OkStatus(); + } + } catch (...) { + } + // Try double + try { + double val; + if (YAML::convert::decode(node, val)) { + proto->set_double_value(val); + return absl::OkStatus(); + } + } catch (...) { + } + // Fallback to string + proto->set_string_value(node.as()); + return absl::OkStatus(); + } + if (node.IsSequence()) { + auto* list = proto->mutable_list_value(); + for (const auto& elem : node) { + CEL_RETURN_IF_ERROR(YamlToExprValue(elem, list->add_values())); + } + return absl::OkStatus(); + } + if (node.IsMap()) { + auto* map_val = proto->mutable_map_value(); + for (auto it = node.begin(); it != node.end(); ++it) { + auto* entry = map_val->add_entries(); + CEL_RETURN_IF_ERROR(YamlToExprValue(it->first, entry->mutable_key())); + CEL_RETURN_IF_ERROR(YamlToExprValue(it->second, entry->mutable_value())); + } + return absl::OkStatus(); + } + return absl::InvalidArgumentError("Unknown YAML node type"); +} + +absl::Status ParseInputValue( + const YAML::Node& node, + cel::expr::conformance::test::InputValue* input_val) { + if (node.IsMap() && node["expr"].IsDefined()) { + input_val->set_expr(node["expr"].as()); + return absl::OkStatus(); + } + if (node.IsMap() && node["value"].IsDefined()) { + return YamlToExprValue(node["value"], input_val->mutable_value()); + } + return YamlToExprValue(node, input_val->mutable_value()); +} + +absl::Status ParseTestOutput(const YAML::Node& node, + cel::expr::conformance::test::TestOutput* output) { + if (!node.IsDefined()) { + return absl::InvalidArgumentError("Missing output node"); + } + if (node.IsMap()) { + if (node["expr"].IsDefined()) { + output->set_result_expr(node["expr"].as()); + return absl::OkStatus(); + } + if (node["value"].IsDefined()) { + return YamlToExprValue(node["value"], output->mutable_result_value()); + } + if (node["error"].IsDefined()) { + auto* eval_error = output->mutable_eval_error(); + eval_error->add_errors()->set_message(node["error"].as()); + return absl::OkStatus(); + } + if (node["error_set"].IsDefined()) { + auto* eval_error = output->mutable_eval_error(); + for (const auto& err : node["error_set"]) { + eval_error->add_errors()->set_message(err.as()); + } + return absl::OkStatus(); + } + if (node["unknown"].IsDefined()) { + auto* unknown = output->mutable_unknown(); + for (const auto& expr_id_node : node["unknown"]) { + unknown->add_exprs(expr_id_node.as()); + } + return absl::OkStatus(); + } + } + return YamlToExprValue(node, output->mutable_result_value()); +} + +absl::StatusOr +ParsePolicyTestSuiteYamlImpl(absl::string_view yaml_content) { + YAML::Node tests_node; + try { + tests_node = YAML::Load(std::string(yaml_content)); + } catch (const std::exception& e) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse YAML: ", e.what())); + } + + cel::expr::conformance::test::TestSuite test_suite; + if (tests_node["description"].IsDefined()) { + test_suite.set_description(tests_node["description"].as()); + } + + YAML::Node sections = tests_node["sections"]; + if (!sections.IsDefined()) { + sections = tests_node["section"]; // support singular format + } + if (!sections.IsDefined()) { + return absl::InvalidArgumentError( + "Missing 'sections' or 'section' in tests YAML"); + } + + for (const auto& section_node : sections) { + auto* section = test_suite.add_sections(); + if (section_node["name"].IsDefined()) { + section->set_name(section_node["name"].as()); + } + if (section_node["description"].IsDefined()) { + section->set_description(section_node["description"].as()); + } + + YAML::Node tests = section_node["tests"]; + if (!tests.IsDefined()) { + tests = section_node["test"]; // support singular format + } + if (!tests.IsDefined()) { + continue; + } + + for (const auto& test_node : tests) { + auto* test_case = section->add_tests(); + if (test_node["name"].IsDefined()) { + test_case->set_name(test_node["name"].as()); + } + if (test_node["description"].IsDefined()) { + test_case->set_description(test_node["description"].as()); + } + if (test_node["context_expr"].IsDefined()) { + test_case->mutable_input_context()->set_context_expr( + test_node["context_expr"].as()); + } + + YAML::Node input_node = test_node["input"]; + if (input_node.IsDefined() && input_node.IsMap()) { + auto* input_map = test_case->mutable_input(); + for (auto it = input_node.begin(); it != input_node.end(); ++it) { + std::string var_name = it->first.as(); + cel::expr::conformance::test::InputValue input_val; + CEL_RETURN_IF_ERROR(ParseInputValue(it->second, &input_val)); + (*input_map)[var_name] = std::move(input_val); + } + } + + YAML::Node output_node = test_node["output"]; + if (output_node.IsDefined()) { + CEL_RETURN_IF_ERROR( + ParseTestOutput(output_node, test_case->mutable_output())); + } + } + } + + return test_suite; +} + +} // namespace + +absl::StatusOr +ParsePolicyTestSuiteYaml(absl::string_view yaml_content) { + try { + return ParsePolicyTestSuiteYamlImpl(yaml_content); + } catch (...) { + return absl::InvalidArgumentError("Failed to parse YAML"); + } +} + +} // namespace cel::test diff --git a/policy/test_util.h b/policy/test_util.h new file mode 100644 index 000000000..5fe306050 --- /dev/null +++ b/policy/test_util.h @@ -0,0 +1,33 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_TEST_UTIL_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_TEST_UTIL_H_ + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "cel/expr/conformance/test/suite.pb.h" + +namespace cel::test { + +// Parses a YAML content representing a policy test suite (tests.yaml) +// and adapts it to the cel.expr.conformance.test.TestSuite protobuf message. +// +// TODO(uncreated-issue/92): Move to the testrunner library. +absl::StatusOr +ParsePolicyTestSuiteYaml(absl::string_view yaml_content); + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_POLICY_TEST_UTIL_H_ diff --git a/policy/testdata/BUILD b/policy/testdata/BUILD new file mode 100644 index 000000000..10a26fa0b --- /dev/null +++ b/policy/testdata/BUILD @@ -0,0 +1,19 @@ +package( + default_testonly = True, + default_visibility = ["//visibility:public"], +) + +filegroup( + name = "policy_testdata", + srcs = glob([ + "*.yaml", + "*.baseline", + ]), +) + +exports_files( + srcs = glob([ + "*.yaml", + "*.baseline", + ]), +) diff --git a/policy/testdata/cel_policy.yaml b/policy/testdata/cel_policy.yaml new file mode 100644 index 000000000..010ad8855 --- /dev/null +++ b/policy/testdata/cel_policy.yaml @@ -0,0 +1,42 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Environment: +# spec: TestAllTypes +name: cel_policy +description: A test policy for CEL +display_name: Cel Policy +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +- name: cel.expr.conformance.proto3.TestAllTypes.NestedEnum +rule: + id: test_rule + description: test rule description + variables: + - name: test_var + expression: > + TestAllTypes{single_int64: 10}.single_int64 + match: + - condition: > + spec.single_int32 > TestAllTypes{single_int64: 10}.single_int64 + output: | + "invalid spec, got single_int32=" + string(spec.single_int32) + ", wanted <= 10" + explanation: | + "invalid spec, spec is greater than 10" + - condition: > + spec.standalone_enum == NestedEnum.BAR + output: | + "invalid spec, reference to BAR is not allowed" + - condition: spec.single_int64 == variables.test_var + output: '"invalid spec: exactly matches test_var"' + explanation: '"the spec cannot have single_int64 set to a known bad value"' \ No newline at end of file diff --git a/policy/testdata/cel_policy_parser.baseline b/policy/testdata/cel_policy_parser.baseline new file mode 100644 index 000000000..7a6678bfe --- /dev/null +++ b/policy/testdata/cel_policy_parser.baseline @@ -0,0 +1,89 @@ +POLICY SOURCE: cel_policy.yaml +-------------------------------------------------------------------- +PARSED POLICY: +CelPolicy{ + =========================================================== + # Copyright 2026 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # https://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # Environment: + # spec: TestAllTypes + #0> name: #1> cel_policy + #2> description: #3> A test policy for CEL + #4> display_name: #5> Cel Policy + #6> imports: + - #7> name: #8> cel.expr.conformance.proto3.TestAllTypes + - #9> name: #10> cel.expr.conformance.proto3.TestAllTypes.NestedEnum + #11> rule: + #13> #12> id: #14> test_rule + #15> description: #16> test rule description + #17> variables: + - #18> name: #19> test_var + #20> expression: #21> > + TestAllTypes{single_int64: 10}.single_int64 + #22> match: + - #24> #23> condition: #25> > + spec.single_int32 > TestAllTypes{single_int64: 10}.single_int64 + #26> output: #27> | + "invalid spec, got single_int32=" + string(spec.single_int32) + ", wanted <= 10" + #28> explanation: #29> | + "invalid spec, spec is greater than 10" + - #31> #30> condition: #32> > + spec.standalone_enum == NestedEnum.BAR + #33> output: #34> | + "invalid spec, reference to BAR is not allowed" + - #36> #35> condition: #37> spec.single_int64 == variables.test_var + #38> output: #39> '"invalid spec: exactly matches test_var"' + #40> explanation: #41> '"the spec cannot have single_int64 set to a known bad value"' + =========================================================== + name: #1> "cel_policy" + description: #3> "A test policy for CEL" + display_name: #5> "Cel Policy" + imports: + #7> name: #8> "cel.expr.conformance.proto3.TestAllTypes" + #9> name: #10> "cel.expr.conformance.proto3.TestAllTypes.NestedEnum" + #12> rule: { + rule_id: #14> "test_rule" + description: #16> "test rule description" + variable: { + name: #19> "test_var" + expression: #21> "TestAllTypes{single_int64: 10}.single_int64 + " + } + #23> match: { + condition: #25> "spec.single_int32 > TestAllTypes{single_int64: 10}.single_int64 + " + result: { + output: #27> ""invalid spec, got single_int32=" + string(spec.single_int32) + ", wanted <= 10" + " + explanation: #29> ""invalid spec, spec is greater than 10" + " + } + } + #30> match: { + condition: #32> "spec.standalone_enum == NestedEnum.BAR + " + result: { + output: #34> ""invalid spec, reference to BAR is not allowed" + " + } + } + #35> match: { + condition: #37> "spec.single_int64 == variables.test_var" + result: { + output: #39> ""invalid spec: exactly matches test_var"" + explanation: #41> ""the spec cannot have single_int64 set to a known bad value"" + } + } + } +} diff --git a/policy/testdata/custom_policy_format.yaml b/policy/testdata/custom_policy_format.yaml new file mode 100644 index 000000000..a67356906 --- /dev/null +++ b/policy/testdata/custom_policy_format.yaml @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: cel_policy_custom_tags +description: A custom policy format +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +purpose: test +version: 42 +conditions: +- if: spec.single_string == "none" + then: "'zero'" + else: + if: spec.single_string == "integer" + then: + if: spec.single_int32 > 0 + then: "'positive integer'" + else: "'negative integer'" + else: "'not an integer'" diff --git a/policy/testdata/custom_policy_format_parser.baseline b/policy/testdata/custom_policy_format_parser.baseline new file mode 100644 index 000000000..d5b1a2235 --- /dev/null +++ b/policy/testdata/custom_policy_format_parser.baseline @@ -0,0 +1,75 @@ +POLICY SOURCE: custom_policy_format.yaml +-------------------------------------------------------------------- +PARSED POLICY: +CelPolicy{ + =========================================================== + # Copyright 2026 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # https://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #0> name: #1> cel_policy_custom_tags + #2> description: #3> A custom policy format + #4> imports: + - #5> name: #6> cel.expr.conformance.proto3.TestAllTypes + #7> purpose: #8> test + #9> version: #10> 42 + #11> conditions: + - #13> #12> if: #14> spec.single_string == "none" + #15> then: #16> "'zero'" + #17> else: + #19> #18> if: #20> spec.single_string == "integer" + #21> then: + #23> #22> if: #24> spec.single_int32 > 0 + #25> then: #26> "'positive integer'" + #27> else: #29> #28> "'negative integer'" + #30> else: #32> #31> "'not an integer'" + + =========================================================== + name: #1> "cel_policy_custom_tags" + description: #3> "A custom policy format" + metadata: { + purpose: #8> "test" + version: 42 + } + imports: + #5> name: #6> "cel.expr.conformance.proto3.TestAllTypes" + rule: { + #12> match: { + condition: #14> "spec.single_string == "none"" + result: { + output: #16> "'zero'" + } + } + #18> match: { + condition: #20> "spec.single_string == "integer"" + result: + rule: { + #22> match: { + condition: #24> "spec.single_int32 > 0" + result: { + output: #26> "'positive integer'" + } + } + #29> match: { + result: { + output: #28> "'negative integer'" + } + } + } + } + #32> match: { + result: { + output: #31> "'not an integer'" + } + } + } +} diff --git a/policy/testdata/custom_policy_format_with_errors.yaml b/policy/testdata/custom_policy_format_with_errors.yaml new file mode 100644 index 000000000..594747c60 --- /dev/null +++ b/policy/testdata/custom_policy_format_with_errors.yaml @@ -0,0 +1,33 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: cel_policy_custom_tags +description: A custom policy format +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +purpose: + - testing +version: new +conditions: +- if: + spec.single_string: "none" + then: "'zero'" + else: "'not zero'" +- if: spec.single_string == "number" + then: + if: spec.single_int32 > 0 + then: "'positive integer'" + else: + - ignore +- else: "'negative integer'" + diff --git a/policy/testdata/custom_policy_format_with_errors_parser.baseline b/policy/testdata/custom_policy_format_with_errors_parser.baseline new file mode 100644 index 000000000..978d27bda --- /dev/null +++ b/policy/testdata/custom_policy_format_with_errors_parser.baseline @@ -0,0 +1,16 @@ +POLICY SOURCE: custom_policy_format_with_errors.yaml +-------------------------------------------------------------------- +-------------------------------------------------------------------- +PARSER ISSUES: +ERROR: custom_policy_format_with_errors.yaml:19:3: Policy purpose is not a string + | - testing + | ..^ +ERROR: custom_policy_format_with_errors.yaml:20:10: Policy version is not an integer: new + | version: new + | .........^ +ERROR: custom_policy_format_with_errors.yaml:23:5: Policy 'if' condition is not a string + | spec.single_string: "none" + | ....^ +ERROR: custom_policy_format_with_errors.yaml:31:7: Bad syntax in 'if/then' block + | - ignore + | ......^ diff --git a/policy/testdata/nested_rule.yaml b/policy/testdata/nested_rule.yaml new file mode 100644 index 000000000..2b07faa64 --- /dev/null +++ b/policy/testdata/nested_rule.yaml @@ -0,0 +1,37 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: nested_rule +rule: + variables: + - name: "permitted_regions" + expression: "['us', 'uk', 'es']" + match: + - rule: + id: "banned regions" + description: > + determine whether the resource origin is in the banned + list. If the region is also in the permitted list, the + ban has no effect. + variables: + - name: "banned_regions" + expression: "{'us': false, 'ru': false, 'ir': false}" + match: + - condition: | + resource.origin in variables.banned_regions && + !(resource.origin in variables.permitted_regions) + output: "{'banned': true}" + - condition: resource.origin in variables.permitted_regions + output: "{'banned': false}" + - output: "{'banned': true}" + explanation: "'resource is in the banned region ' + resource.origin" \ No newline at end of file diff --git a/policy/testdata/nested_rule_parser.baseline b/policy/testdata/nested_rule_parser.baseline new file mode 100644 index 000000000..128f81bda --- /dev/null +++ b/policy/testdata/nested_rule_parser.baseline @@ -0,0 +1,84 @@ +POLICY SOURCE: nested_rule.yaml +-------------------------------------------------------------------- +PARSED POLICY: +CelPolicy{ + =========================================================== + # Copyright 2024 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # https://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #0> name: #1> nested_rule + #2> rule: + #4> #3> variables: + - #5> name: #6> "permitted_regions" + #7> expression: #8> "['us', 'uk', 'es']" + #9> match: + - #11> #10> rule: + #13> #12> id: #14> "banned regions" + #15> description: #16> > + determine whether the resource origin is in the banned + list. If the region is also in the permitted list, the + ban has no effect. + #17> variables: + - #18> name: #19> "banned_regions" + #20> expression: #21> "{'us': false, 'ru': false, 'ir': false}" + #22> match: + - #24> #23> condition: #25> | + resource.origin in variables.banned_regions && + !(resource.origin in variables.permitted_regions) + #26> output: #27> "{'banned': true}" + - #29> #28> condition: #30> resource.origin in variables.permitted_regions + #31> output: #32> "{'banned': false}" + - #34> #33> output: #35> "{'banned': true}" + #36> explanation: #37> "'resource is in the banned region ' + resource.origin" + =========================================================== + name: #1> "nested_rule" + description: "nested_rule.yaml" + #3> rule: { + variable: { + name: #6> "permitted_regions" + expression: #8> "['us', 'uk', 'es']" + } + #10> match: { + result: + #12> rule: { + rule_id: #14> "banned regions" + description: #16> "determine whether the resource origin is in the banned list. If the region is also in the permitted list, the ban has no effect. + " + variable: { + name: #19> "banned_regions" + expression: #21> "{'us': false, 'ru': false, 'ir': false}" + } + #23> match: { + condition: #25> "resource.origin in variables.banned_regions && + !(resource.origin in variables.permitted_regions) + " + result: { + output: #27> "{'banned': true}" + } + } + } + } + #28> match: { + condition: #30> "resource.origin in variables.permitted_regions" + result: { + output: #32> "{'banned': false}" + } + } + #33> match: { + result: { + output: #35> "{'banned': true}" + explanation: #37> "'resource is in the banned region ' + resource.origin" + } + } + } +} diff --git a/policy/yaml_policy_parser.cc b/policy/yaml_policy_parser.cc new file mode 100644 index 000000000..c838cff33 --- /dev/null +++ b/policy/yaml_policy_parser.cc @@ -0,0 +1,411 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/yaml_policy_parser.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_context.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_parser.h" +#include "yaml-cpp/exceptions.h" +#include "yaml-cpp/node/node.h" +#include "yaml-cpp/node/parse.h" +#include "yaml-cpp/null.h" +#include "yaml-cpp/yaml.h" // IWYU pragma: keep + +namespace cel { + +CelPolicyElementId YamlPolicyParser::CollectMetadata( + CelPolicyParseContext& ctx, const YAML::Node& node) const { + CelPolicyElementId element_id = ctx.next_element_id(); + if (!node.Mark().is_null()) { + ctx.policy_source().NoteSourcePosition(element_id, node.Mark().pos); + } + return element_id; +} + +std::optional YamlPolicyParser::GetValueString( + CelPolicyParseContext& ctx, const YAML::Node& node, + std::string_view error_message) const { + if (!node.IsDefined()) { + // This should never happen since the YAML syntax has already been checked. + return std::nullopt; + } + + CelPolicyElementId id = CollectMetadata(ctx, node); + if (!node.IsScalar()) { + ctx.ReportError(id, error_message); + return std::nullopt; + } + + try { + return ValueString(id, node.as()); + } catch (YAML::Exception& e) { + // This should never happen since we already checked that the node is a + // scalar and all scalars can be converted to strings. + return std::nullopt; + } +} + +absl::Status YamlPolicyParser::ParsePolicy(CelPolicyParseContext& ctx) const { + const Source* source = ctx.policy_source().content(); + if (source == nullptr) { + return absl::OkStatus(); + } + + ctx.policy().set_description(ValueString(-1, source->description())); + std::string text = source->content().ToString(); + YAML::Node node; + try { + node = YAML::Load(text); + } catch (YAML::Exception& e) { + if (!e.mark.is_null()) { + ctx.policy_source().NoteSourcePosition(0, e.mark.pos); + } + ctx.ReportError(0, "Invalid CEL policy YAML syntax"); + return absl::OkStatus(); + } + + if (!node.IsMap()) { + ctx.ReportError(CollectMetadata(ctx, node), "Policy is not a map"); + return absl::OkStatus(); + } + + for (auto it = node.begin(); it != node.end(); ++it) { + const YAML::Node key_node = it->first; + const YAML::Node value_node = it->second; + std::optional key = + GetValueString(ctx, key_node, "Policy tag is not a string"); + if (!key.has_value()) { + continue; + } + CEL_ASSIGN_OR_RETURN(bool handled, ParsePolicyTag(ctx, *key, value_node)); + if (!handled) { + ctx.ReportError( + key->id(), + absl::StrCat("Unrecognized top-level policy tag: ", key->value())); + } + } + + return absl::OkStatus(); +} + +absl::StatusOr YamlPolicyParser::ParsePolicyTag( + CelPolicyParseContext& ctx, const ValueString& tag_name, + const YAML::Node& node) const { + if (tag_name.value() == "imports") { + CEL_RETURN_IF_ERROR(ParseImports(ctx, node)); + return true; + } + if (tag_name.value() == "name") { + std::optional name = + GetValueString(ctx, node, "Policy 'name' is not a string"); + if (name.has_value()) { + ctx.policy().set_name(*name); + } + return true; + } + if (tag_name.value() == "description") { + std::optional description = + GetValueString(ctx, node, "Policy 'description' is not a string"); + if (description.has_value()) { + ctx.policy().set_description(*description); + } + return true; + } + if (tag_name.value() == "display_name") { + std::optional display_name = + GetValueString(ctx, node, "Policy 'display_name' is not a string"); + if (display_name.has_value()) { + ctx.policy().set_display_name(*display_name); + } + return true; + } + if (tag_name.value() == "rule") { + CEL_RETURN_IF_ERROR(ParseRule(ctx, node, ctx.policy().mutable_rule())); + return true; + } + return false; +} + +absl::Status YamlPolicyParser::ParseImports(CelPolicyParseContext& ctx, + const YAML::Node& node) const { + if (!node.IsSequence()) { + ctx.ReportError(CollectMetadata(ctx, node), + "Policy 'imports' is not a sequence"); + return absl::OkStatus(); + } + + for (const YAML::Node& import : node) { + CelPolicyElementId import_id = CollectMetadata(ctx, import); + if (!import.IsMap()) { + ctx.ReportError(import_id, "Import is not a map"); + continue; + } + const YAML::Node& name_node = import["name"]; + if (!name_node.IsDefined()) { + ctx.ReportError(import_id, "No 'name' tag in import"); + continue; + } + std::optional import_name = + GetValueString(ctx, name_node, "Import name is not a string"); + if (import_name.has_value()) { + ctx.policy().mutable_imports().push_back(Import(import_id, *import_name)); + } + } + return absl::OkStatus(); +} + +absl::Status YamlPolicyParser::ParseRule(CelPolicyParseContext& ctx, + const YAML::Node& node, + Rule& rule) const { + if (!node.IsMap()) { + ctx.ReportError(CollectMetadata(ctx, node), "Policy 'rule' is not a map"); + return absl::OkStatus(); + } + rule.set_id(CollectMetadata(ctx, node)); + + for (auto it = node.begin(); it != node.end(); ++it) { + const YAML::Node key_node = it->first; + const YAML::Node value_node = it->second; + std::optional key = + GetValueString(ctx, key_node, "Policy rule tag is not a string"); + if (!key.has_value()) { + continue; + } + CEL_ASSIGN_OR_RETURN(bool handled, + ParseRuleTag(ctx, *key, value_node, rule)); + if (!handled) { + ctx.ReportError(key->id(), absl::StrCat("Unrecognized policy rule tag: ", + key->value())); + } + } + return absl::OkStatus(); +} + +absl::StatusOr YamlPolicyParser::ParseRuleTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, + Rule& rule) const { + if (tag_name.value() == "id") { + std::optional rule_id = + GetValueString(ctx, node, "Policy rule 'id' is not a string"); + if (rule_id.has_value()) { + rule.set_rule_id(*rule_id); + } + return true; + } + if (tag_name.value() == "description") { + std::optional description = + GetValueString(ctx, node, "Policy rule 'description' is not a string"); + if (description.has_value()) { + rule.set_description(*description); + } + return true; + } + if (tag_name.value() == "variables") { + if (!node.IsSequence()) { + ctx.ReportError(CollectMetadata(ctx, node), + "Policy rule 'variables' is not a sequence"); + return true; + } + for (const YAML::Node& variable_node : node) { + CEL_ASSIGN_OR_RETURN(Variable variable, + ParseVariable(ctx, variable_node, rule)); + rule.mutable_variables().push_back(std::move(variable)); + } + return true; + } + if (tag_name.value() == "match") { + if (!node.IsSequence()) { + ctx.ReportError(CollectMetadata(ctx, node), + "Policy rule 'match' is not a sequence"); + return true; + } + for (const YAML::Node& match_node : node) { + CEL_ASSIGN_OR_RETURN(Match match, ParseMatch(ctx, match_node, rule)); + rule.mutable_matches().push_back(std::move(match)); + } + return true; + } + return false; +} + +absl::StatusOr YamlPolicyParser::ParseVariable( + CelPolicyParseContext& ctx, const YAML::Node& node, Rule& rule) const { + Variable variable; + if (!node.IsMap()) { + ctx.ReportError(CollectMetadata(ctx, node), + "Policy rule 'variable' is not a map"); + return variable; + } + for (auto it = node.begin(); it != node.end(); ++it) { + const YAML::Node key_node = it->first; + const YAML::Node value_node = it->second; + std::optional key = + GetValueString(ctx, key_node, "Policy variable tag is not a string"); + if (!key.has_value()) { + continue; + } + CEL_ASSIGN_OR_RETURN(bool handled, + ParseVariableTag(ctx, *key, value_node, variable)); + if (!handled) { + ctx.ReportError( + key->id(), + absl::StrCat("Unrecognized policy variable tag: ", key->value())); + } + } + return variable; +} + +absl::StatusOr YamlPolicyParser::ParseVariableTag( + CelPolicyParseContext& ctx, const ValueString& tag_name, + const YAML::Node& node, Variable& variable) const { + if (tag_name.value() == "name") { + std::optional name = + GetValueString(ctx, node, "Policy variable 'name' is not a string"); + if (name.has_value()) { + variable.set_name(*name); + } + return true; + } + if (tag_name.value() == "expression") { + std::optional expression = GetValueString( + ctx, node, "Policy variable 'expression' is not a string"); + if (expression.has_value()) { + variable.set_expression(*expression); + } + return true; + } + return false; +} + +absl::StatusOr YamlPolicyParser::ParseMatch(CelPolicyParseContext& ctx, + const YAML::Node& node, + Rule& rule) const { + Match match; + match.set_id(CollectMetadata(ctx, node)); + if (!node.IsMap()) { + ctx.ReportError(match.id(), "Policy rule 'match' is not a map"); + return match; + } + for (auto it = node.begin(); it != node.end(); ++it) { + const YAML::Node key_node = it->first; + const YAML::Node value_node = it->second; + std::optional key = + GetValueString(ctx, key_node, "Policy match tag is not a string"); + if (!key.has_value()) { + continue; + } + CEL_ASSIGN_OR_RETURN(bool handled, + ParseMatchTag(ctx, *key, value_node, match, rule)); + if (!handled) { + ctx.ReportError(key->id(), absl::StrCat("Unrecognized policy match tag: ", + key->value())); + } + } + + if (match.has_output_block()) { + if (match.output_block().output().value().empty() && + match.output_block().explanation().has_value()) { + ctx.ReportError(match.id(), "Match specifies explanation but no output"); + } + } + + return match; +} + +absl::StatusOr YamlPolicyParser::ParseMatchTag( + CelPolicyParseContext& ctx, const ValueString& tag_name, + const YAML::Node& node, Match& match, Rule& rule) const { + if (tag_name.value() == "condition") { + std::optional condition = + GetValueString(ctx, node, "Policy match 'condition' is not a string"); + if (condition.has_value()) { + match.set_condition(*condition); + } + return true; + } + if (tag_name.value() == "explanation") { + std::optional explanation = + GetValueString(ctx, node, "Policy match 'explanation' is not a string"); + if (explanation.has_value()) { + if (match.has_rule()) { + ctx.ReportError( + tag_name.id(), + "Cannot specify explanation when a nested rule is present"); + } else { + match.mutable_output_block().set_explanation(*explanation); + } + } + return true; + } + if (tag_name.value() == "output") { + std::optional output = + GetValueString(ctx, node, "Policy match 'output' is not a string"); + if (output.has_value()) { + if (match.has_rule()) { + ctx.ReportError(tag_name.id(), + "Cannot specify output when a nested rule is present"); + } else { + match.mutable_output_block().set_output(*output); + } + } + return true; + } + if (tag_name.value() == "rule") { + if (match.has_output_block()) { + ctx.ReportError(tag_name.id(), + "Cannot specify nested rule when output/explanation is " + "present"); + } + auto nested_rule = std::make_unique(); + CEL_RETURN_IF_ERROR(ParseRule(ctx, node, *nested_rule)); + match.set_result(std::move(nested_rule)); + return true; + } + return false; +} + +const CelPolicyParser& GetDefaultYamlPolicyParser() { + static const auto* const parser = new YamlPolicyParser(); + return *parser; +} + +absl::StatusOr ParseYamlCelPolicy( + std::shared_ptr policy_source) { + return ParseYamlCelPolicy(std::move(policy_source), + GetDefaultYamlPolicyParser()); +} + +absl::StatusOr ParseYamlCelPolicy( + std::shared_ptr policy_source, + const CelPolicyParser& parser) { + CelPolicyParseContext ctx(std::move(policy_source)); + CEL_RETURN_IF_ERROR(parser.ParsePolicy(ctx)); + return ctx.GetResult(); +} + +} // namespace cel diff --git a/policy/yaml_policy_parser.h b/policy/yaml_policy_parser.h new file mode 100644 index 000000000..469209333 --- /dev/null +++ b/policy/yaml_policy_parser.h @@ -0,0 +1,135 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_YAML_POLICY_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_YAML_POLICY_PARSER_H_ + +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_context.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_parser.h" +#include "yaml-cpp/node/node.h" + +namespace cel { + +// A parser for YAML-based CEL policies. +// +// To support additional or alternative YAML elements, subclass +// `YamlPolicyParser` and override specific parsing methods, `Parse*` +class YamlPolicyParser : public CelPolicyParser { + public: + std::optional GetValueString( + CelPolicyParseContext& ctx, const YAML::Node& node, + std::string_view error_message) const; + + absl::Status ParsePolicy(CelPolicyParseContext& ctx) const override; + + protected: + // Collects metadata (e.g. source position) for the given YAML node, stores it + // in the context, and returns an ID that can be used to refer to it. + virtual CelPolicyElementId CollectMetadata(CelPolicyParseContext& ctx, + const YAML::Node& node) const; + + // Parses a top-level tag in the policy YAML. + // Returns true if the tag was handled. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParsePolicyTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node) const; + + // Parses the imports section of the policy YAML. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::Status ParseImports(CelPolicyParseContext& ctx, + const YAML::Node& node) const; + + // Parses a rule element of the policy YAML, which may be the top-level rule + // or a sub-rule of a match. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::Status ParseRule(CelPolicyParseContext& ctx, + const YAML::Node& node, Rule& rule) const; + + // Parses a tag in a policy YAML rule. + // Returns true if the tag was handled. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseRuleTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, + Rule& rule) const; + + // Parses a variable element of the policy YAML. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseVariable(CelPolicyParseContext& ctx, + const YAML::Node& node, + Rule& rule) const; + + // Parses a tag in a policy YAML variable. + // Returns true if the tag was handled. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseVariableTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, + Variable& variable) const; + + // Parses a match element of the policy YAML. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseMatch(CelPolicyParseContext& ctx, + const YAML::Node& node, + Rule& rule) const; + + // Parses a tag in a policy YAML match. + // Returns true if the tag was handled. + // + // Note that an OkStatus does not necessarily mean that parsing was successful + // - only that it can continue. + virtual absl::StatusOr ParseMatchTag(CelPolicyParseContext& ctx, + const ValueString& tag_name, + const YAML::Node& node, + Match& match, Rule& rule) const; +}; + +// Returns a default implementation of YamlPolicyParser. +const CelPolicyParser& GetDefaultYamlPolicyParser(); + +absl::StatusOr ParseYamlCelPolicy( + std::shared_ptr policy_source, + const CelPolicyParser& parser); + +// YAML CelPolicy parser that uses the default format as implemented by +// `YamlPolicyParser`. +absl::StatusOr ParseYamlCelPolicy( + std::shared_ptr policy_source); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_POLICY_YAML_POLICY_PARSER_H_ diff --git a/policy/yaml_policy_parser_test.cc b/policy/yaml_policy_parser_test.cc new file mode 100644 index 000000000..4e7dfc49c --- /dev/null +++ b/policy/yaml_policy_parser_test.cc @@ -0,0 +1,305 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/yaml_policy_parser.h" + +#include +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "internal/runfiles.h" +#include "internal/testing.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_parser.h" +#include "yaml-cpp/node/node.h" + +namespace cel { + +namespace internal { +const CelPolicyParser& GetTestCustomYamlPolicyParser(); +} // namespace internal + +namespace { + +using ::absl_testing::IsOk; +using ::testing::HasSubstr; +using ::testing::IsNull; + +constexpr absl::string_view kTestPolicyFilePath = +"_main/policy/testdata/"; + +constexpr absl::string_view kBaselineSeparator = + "--------------------------------------------------------------------\n"; + +struct YamlPolicyParserTestCase { + std::string policy_source_file; + std::string baseline_file; + const cel::CelPolicyParser& (*parser_factory)(); +}; + +using YamlPolicyParserTest = testing::TestWithParam; + +TEST_P(YamlPolicyParserTest, Parse) { + std::string contents; + std::string test_file = cel::internal::ResolveRunfilesPath( + absl::StrCat(kTestPolicyFilePath, GetParam().policy_source_file)); + ASSERT_THAT(cel::internal::GetFileContents(test_file, &contents), IsOk()); + + std::string baseline; + std::string baseline_file = cel::internal::ResolveRunfilesPath( + absl::StrCat(kTestPolicyFilePath, GetParam().baseline_file)); + ASSERT_THAT(cel::internal::GetFileContents(baseline_file, &baseline), IsOk()); + baseline = absl::StripAsciiWhitespace(baseline); + + std::ostringstream out; + out << "POLICY SOURCE: " << GetParam().policy_source_file << "\n"; + + ASSERT_OK_AND_ASSIGN(cel::SourcePtr source, + cel::NewSource(contents, GetParam().policy_source_file)); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + + ASSERT_OK_AND_ASSIGN( + CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source, GetParam().parser_factory())); + + out << kBaselineSeparator; + if (parse_result.IsValid()) { + out << "PARSED POLICY:\n"; + out << parse_result.GetPolicy()->DebugString(); + } else { + ASSERT_THAT(parse_result.GetPolicy(), IsNull()); + out << kBaselineSeparator; + out << "PARSER ISSUES:\n"; + for (const auto& issue : parse_result.GetIssues()) { + out << issue.ToDisplayString(*policy_source) << "\n"; + } + } + + std::string actual(absl::StripAsciiWhitespace(out.str())); + if (actual != baseline) { + // Log the actual result to make it easier to copy/paste into the baseline + // file when updating the tests. + ABSL_LOG(INFO) << "Actual:\n" << actual; + EXPECT_EQ(actual, baseline); + } +} + +INSTANTIATE_TEST_SUITE_P( + Formats, YamlPolicyParserTest, + testing::ValuesIn({ + YamlPolicyParserTestCase{ + .policy_source_file = "cel_policy.yaml", + .baseline_file = "cel_policy_parser.baseline", + .parser_factory = GetDefaultYamlPolicyParser, + }, + YamlPolicyParserTestCase{ + .policy_source_file = "nested_rule.yaml", + .baseline_file = "nested_rule_parser.baseline", + .parser_factory = GetDefaultYamlPolicyParser, + }, + YamlPolicyParserTestCase{ + .policy_source_file = "custom_policy_format.yaml", + .baseline_file = "custom_policy_format_parser.baseline", + .parser_factory = internal::GetTestCustomYamlPolicyParser, + }, + YamlPolicyParserTestCase{ + .policy_source_file = "custom_policy_format_with_errors.yaml", + .baseline_file = "custom_policy_format_with_errors_parser.baseline", + .parser_factory = internal::GetTestCustomYamlPolicyParser, + }, + })); + +struct ParseTestCase { + std::string yaml; + std::string expected_error; +}; + +using YamlPolicyParseErrorTest = testing::TestWithParam; + +TEST_P(YamlPolicyParseErrorTest, YamlSyntaxError) { + const ParseTestCase& param = GetParam(); + ASSERT_OK_AND_ASSIGN(cel::SourcePtr source, + cel::NewSource(param.yaml, "test")); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + ASSERT_OK_AND_ASSIGN(CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source)); + EXPECT_THAT(parse_result.FormattedIssues(), HasSubstr(param.expected_error)); +} + +std::vector GetParseTestCases() { + return { + ParseTestCase{ + .yaml = R"yaml( ? [ John, Doe ]: age: 30 )yaml", + .expected_error = "1:22: Invalid CEL policy YAML syntax\n" + " | ? [ John, Doe ]: age: 30 \n" + " | .....................^", + }, + ParseTestCase{ + .yaml = R"yaml( invalid yaml )yaml", + .expected_error = "1:2: Policy is not a map\n" + " | invalid yaml \n" + " | .^", + }, + ParseTestCase{ + .yaml = R"yaml( + ? [1, 2, 3] + : "Prime numbers sequence" + )yaml", + .expected_error = "2:23: Policy tag is not a string\n" + " | ? [1, 2, 3]\n" + " | ......................^", + }, + ParseTestCase{ + .yaml = R"yaml( + imports: N/A + )yaml", + .expected_error = "2:28: Policy 'imports' is not a sequence\n" + " | imports: N/A\n" + " | ...........................^", + }, + ParseTestCase{ + .yaml = R"yaml( + imports: + - cel.expr.conformance + )yaml", + .expected_error = "3:21: Import is not a map\n" + " | - cel.expr.conformance\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + imports: + - name: + - cel.expr.conformance + )yaml", + .expected_error = "4:21: Import name is not a string\n" + " | - cel.expr.conformance\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: do something + )yaml", + .expected_error = "2:25: Policy 'rule' is not a map\n" + " | rule: do something\n" + " | ........................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + id: + - 22 + )yaml", + .expected_error = "4:21: Policy rule 'id' is not a string\n" + " | - 22\n" + " | ....................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + variables: + no vars + )yaml", + .expected_error = "4:23: Policy rule 'variables' is not a sequence\n" + " | no vars\n" + " | ......................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + variables: + - name: + foo: bar + )yaml", + .expected_error = "5:25: Policy variable 'name' is not a string\n" + " | foo: bar\n" + " | ........................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + variables: + - name: test_var + expression: + - 22 + )yaml", + .expected_error = + "6:23: Policy variable 'expression' is not a string\n" + " | - 22\n" + " | ......................^", + }, + ParseTestCase{ + .yaml = R"yaml( + rule: + variables: + - name: '\u0041\u00a9\u20ac\U0001f680' + - '\u0041\u00a9\u20ac\U0001f680': name + )yaml", + .expected_error = + "5:23: Unrecognized policy variable tag: " + "\\u0041\\u00a9\\u20ac\\U0001f680\n" + " | - '\\u0041\\u00a9\\u20ac\\U0001f680': " + "name\n" + " | ......................^", + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(YamlPolicyParseErrorTest, YamlPolicyParseErrorTest, + ::testing::ValuesIn(GetParseTestCases())); + +TEST(YamlPolicyParserTest, OffsetIssueFormatting) { + // TODO(b/506179116): will need to copy the go implementation in extracting + // the source string from the YAML document instead of the interpreted string + // value to fix up error locations in folded and block literals. + std::string contents; + std::string test_file = cel::internal::ResolveRunfilesPath( + absl::StrCat(kTestPolicyFilePath, "cel_policy.yaml")); + ASSERT_THAT(cel::internal::GetFileContents(test_file, &contents), IsOk()); + + ASSERT_OK_AND_ASSIGN(cel::SourcePtr source, + cel::NewSource(contents, "cel_policy.yaml")); + std::shared_ptr policy_source = + std::make_shared(std::move(source)); + ASSERT_OK_AND_ASSIGN(CelPolicyParseResult parse_result, + cel::ParseYamlCelPolicy(policy_source)); + + ASSERT_TRUE(parse_result.IsValid()); + const CelPolicy* policy = parse_result.GetPolicy(); + + CelPolicyElementId name_id = policy->name().id(); + + CelPolicyIssue issue(name_id, 4, CelPolicyIssue::Severity::kError, + "Test error"); + + std::string formatted = issue.ToDisplayString(*policy_source); + + EXPECT_THAT(formatted, HasSubstr("ERROR: cel_policy.yaml:16:11: Test error")); + EXPECT_THAT(formatted, HasSubstr(" | name: cel_policy")); + EXPECT_THAT(formatted, HasSubstr(" | ..........^")); +} + +} // namespace +} // namespace cel From c8c187fea145b3113b3638d76c727c9052f13511 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 16 Jun 2026 10:10:54 -0700 Subject: [PATCH 067/350] Support overload lookup and filtering by signature as an alternative to overload_id PiperOrigin-RevId: 933154803 --- checker/BUILD | 2 + checker/internal/type_checker_builder_impl.cc | 4 +- checker/type_checker_builder.h | 2 +- checker/type_checker_builder_factory_test.cc | 17 +++-- checker/type_checker_subset_factory.cc | 20 ++++-- checker/type_checker_subset_factory_test.cc | 33 +++++++-- common/BUILD | 3 +- common/decl.cc | 68 +++++++++++-------- common/decl.h | 48 +++---------- common/decl_test.cc | 47 +++++++++++++ env/BUILD | 1 + env/config_test.cc | 55 +++++++++++++++ env/env.cc | 56 +++++++++++---- env/env_test.cc | 16 +++++ env/env_yaml_test.cc | 12 ++-- 15 files changed, 276 insertions(+), 108 deletions(-) diff --git a/checker/BUILD b/checker/BUILD index 27a1eb84e..10ed6e363 100644 --- a/checker/BUILD +++ b/checker/BUILD @@ -229,6 +229,8 @@ cc_library( hdrs = ["type_checker_subset_factory.h"], deps = [ ":type_checker_builder", + "//common:decl", + "//common/internal:signature", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:span", diff --git a/checker/internal/type_checker_builder_impl.cc b/checker/internal/type_checker_builder_impl.cc index 9b91fc926..f0332b999 100644 --- a/checker/internal/type_checker_builder_impl.cc +++ b/checker/internal/type_checker_builder_impl.cc @@ -187,8 +187,8 @@ std::optional FilterDecl(FunctionDecl decl, FunctionDecl filtered; std::string name = decl.release_name(); std::vector overloads = decl.release_overloads(); - for (const auto& ovl : overloads) { - if (subset.should_include_overload(name, ovl.id())) { + for (auto& ovl : overloads) { + if (subset.should_include_overload(name, ovl)) { absl::Status s = filtered.AddOverload(std::move(ovl)); if (!s.ok()) { // Should not be possible to construct the original decl in a way that diff --git a/checker/type_checker_builder.h b/checker/type_checker_builder.h index f145b8a98..c2d0cbf7b 100644 --- a/checker/type_checker_builder.h +++ b/checker/type_checker_builder.h @@ -52,7 +52,7 @@ struct CheckerLibrary { // Represents a declaration to only use a subset of a library. struct TypeCheckerSubset { using FunctionPredicate = absl::AnyInvocable; + absl::string_view function, const OverloadDecl& overload) const>; // The id of the library to subset. Only one subset can be applied per // library id. diff --git a/checker/type_checker_builder_factory_test.cc b/checker/type_checker_builder_factory_test.cc index 9c4775e7f..40406948d 100644 --- a/checker/type_checker_builder_factory_test.cc +++ b/checker/type_checker_builder_factory_test.cc @@ -235,8 +235,8 @@ TEST(TypeCheckerBuilderTest, AddLibraryIncludeSubset) { ASSERT_THAT( builder->AddLibrarySubset( {"testlib", - [](absl::string_view /*function*/, absl::string_view overload_id) { - return (overload_id == "add_int" || overload_id == "sub_int"); + [](absl::string_view /*function*/, const OverloadDecl& overload) { + return (overload.id() == "add_int" || overload.id() == "sub_int"); }}), IsOk()); ASSERT_OK_AND_ASSIGN(auto checker, builder->Build()); @@ -274,9 +274,8 @@ TEST(TypeCheckerBuilderTest, AddLibraryExcludeSubset) { ASSERT_THAT( builder->AddLibrarySubset( {"testlib", - [](absl::string_view /*function*/, absl::string_view overload_id) { - return (overload_id != "add_int" && overload_id != "sub_int"); - ; + [](absl::string_view /*function*/, const OverloadDecl& overload) { + return (overload.id() != "add_int" && overload.id() != "sub_int"); }}), IsOk()); ASSERT_OK_AND_ASSIGN(auto checker, builder->Build()); @@ -313,7 +312,7 @@ TEST(TypeCheckerBuilderTest, AddLibrarySubsetRemoveAllOvl) { ASSERT_THAT(builder->AddLibrary(SubsetTestlib()), IsOk()); ASSERT_THAT(builder->AddLibrarySubset({"testlib", [](absl::string_view function, - absl::string_view /*overload_id*/) { + const OverloadDecl& /*overload*/) { return function != "add"; }}), IsOk()); @@ -352,12 +351,12 @@ TEST(TypeCheckerBuilderTest, AddLibraryOneSubsetPerLibraryId) { ASSERT_THAT( builder->AddLibrarySubset( {"testlib", [](absl::string_view function, - absl::string_view /*overload_id*/) { return true; }}), + const OverloadDecl& /*overload*/) { return true; }}), IsOk()); EXPECT_THAT( builder->AddLibrarySubset( {"testlib", [](absl::string_view function, - absl::string_view /*overload_id*/) { return true; }}), + const OverloadDecl& /*overload*/) { return true; }}), StatusIs(absl::StatusCode::kAlreadyExists)); } @@ -369,7 +368,7 @@ TEST(TypeCheckerBuilderTest, AddLibrarySubsetLibraryIdRequireds) { ASSERT_THAT(builder->AddLibrary(SubsetTestlib()), IsOk()); EXPECT_THAT(builder->AddLibrarySubset({"", [](absl::string_view function, - absl::string_view /*overload_id*/) { + const OverloadDecl& /*overload*/) { return function == "add"; }}), StatusIs(absl::StatusCode::kInvalidArgument)); diff --git a/checker/type_checker_subset_factory.cc b/checker/type_checker_subset_factory.cc index 6a05ce220..e5335e220 100644 --- a/checker/type_checker_subset_factory.cc +++ b/checker/type_checker_subset_factory.cc @@ -21,14 +21,21 @@ #include "absl/strings/string_view.h" #include "absl/types/span.h" #include "checker/type_checker_builder.h" +#include "common/decl.h" +#include "common/internal/signature.h" namespace cel { TypeCheckerSubset::FunctionPredicate IncludeOverloadsByIdPredicate( absl::flat_hash_set overload_ids) { return [overload_ids = std::move(overload_ids)]( - absl::string_view /*function*/, absl::string_view overload_id) { - return overload_ids.contains(overload_id); + absl::string_view function, const OverloadDecl& overload) { + if (overload_ids.contains(overload.id())) { + return true; + } + auto signature = common_internal::MakeOverloadSignature( + function, overload.args(), overload.member()); + return signature.ok() && overload_ids.contains(*signature); }; } @@ -41,8 +48,13 @@ TypeCheckerSubset::FunctionPredicate IncludeOverloadsByIdPredicate( TypeCheckerSubset::FunctionPredicate ExcludeOverloadsByIdPredicate( absl::flat_hash_set overload_ids) { return [overload_ids = std::move(overload_ids)]( - absl::string_view /*function*/, absl::string_view overload_id) { - return !overload_ids.contains(overload_id); + absl::string_view function, const OverloadDecl& overload) { + if (overload_ids.contains(overload.id())) { + return false; + } + auto signature = common_internal::MakeOverloadSignature( + function, overload.args(), overload.member()); + return !signature.ok() || !overload_ids.contains(*signature); }; } diff --git a/checker/type_checker_subset_factory_test.cc b/checker/type_checker_subset_factory_test.cc index fa38e1c0d..5b644ec7c 100644 --- a/checker/type_checker_subset_factory_test.cc +++ b/checker/type_checker_subset_factory_test.cc @@ -43,6 +43,8 @@ TEST(TypeCheckerSubsetFactoryTest, IncludeOverloadsByIdPredicate) { StandardOverloadIds::kEquals, StandardOverloadIds::kNotEquals, StandardOverloadIds::kNotStrictlyFalse, + "matches(string,string)", + "string.matches(string)", }; ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); ASSERT_THAT(builder->GetCheckerBuilder().AddLibrarySubset({ @@ -65,15 +67,19 @@ TEST(TypeCheckerSubsetFactoryTest, IncludeOverloadsByIdPredicate) { EXPECT_TRUE(r.IsValid()); + // Allowed by signature. + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("r'foo.*'.matches('foobar')")); + EXPECT_TRUE(r.IsValid()); + + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("matches(r'foo.*', 'foobar')")); + EXPECT_TRUE(r.IsValid()); + // Not in allowlist. ASSERT_OK_AND_ASSIGN(r, compiler->Compile("1 + 2 < 3")); EXPECT_FALSE(r.IsValid()); ASSERT_OK_AND_ASSIGN(r, compiler->Compile("'abc' + 'def'")); EXPECT_FALSE(r.IsValid()); - - ASSERT_OK_AND_ASSIGN(r, compiler->Compile("r'foo.*'.matches('foobar')")); - EXPECT_FALSE(r.IsValid()); } TEST(TypeCheckerSubsetFactoryTest, ExcludeOverloadsByIdPredicate) { @@ -83,6 +89,8 @@ TEST(TypeCheckerSubsetFactoryTest, ExcludeOverloadsByIdPredicate) { absl::string_view exclude_list[] = { StandardOverloadIds::kMatches, StandardOverloadIds::kMatchesMember, + "size(string)", + "string.size()", }; ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); ASSERT_THAT(builder->GetCheckerBuilder().AddLibrarySubset({ @@ -105,18 +113,35 @@ TEST(TypeCheckerSubsetFactoryTest, ExcludeOverloadsByIdPredicate) { EXPECT_TRUE(r.IsValid()); - // Not in allowlist. + // Allowed. ASSERT_OK_AND_ASSIGN(r, compiler->Compile("1 + 2 < 3")); EXPECT_TRUE(r.IsValid()); ASSERT_OK_AND_ASSIGN(r, compiler->Compile("'abc' + 'def'")); EXPECT_TRUE(r.IsValid()); + // Excluded by ID. ASSERT_OK_AND_ASSIGN(r, compiler->Compile("r'foo.*'.matches('foobar')")); EXPECT_FALSE(r.IsValid()); ASSERT_OK_AND_ASSIGN(r, compiler->Compile("matches(r'foo.*', 'foobar')")); EXPECT_FALSE(r.IsValid()); + + // Excluded by signature (top-level function). + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("size('abc')")); + EXPECT_FALSE(r.IsValid()); + + // Allowed (other overloads of size). + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("size([1, 2, 3])")); + EXPECT_TRUE(r.IsValid()); + + // Excluded by signature (member function). + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("'abc'.size()")); + EXPECT_FALSE(r.IsValid()); + + // Allowed (other overloads of size member). + ASSERT_OK_AND_ASSIGN(r, compiler->Compile("[1, 2, 3].size()")); + EXPECT_TRUE(r.IsValid()); } } // namespace diff --git a/common/BUILD b/common/BUILD index 93410306f..0bd3632dd 100644 --- a/common/BUILD +++ b/common/BUILD @@ -151,9 +151,8 @@ cc_library( "//internal:status_macros", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/hash", - "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/common/decl.cc b/common/decl.cc index b338bfd4f..d2d50964a 100644 --- a/common/decl.cc +++ b/common/decl.cc @@ -20,8 +20,8 @@ #include #include +#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" -#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" @@ -109,43 +109,46 @@ bool SignaturesOverlap(const OverloadDecl& lhs, const OverloadDecl& rhs) { template void AddOverloadInternal(std::string_view function_name, std::vector& insertion_order, - OverloadDeclHashSet& overloads, Overload&& overload, - absl::Status& status) { + absl::flat_hash_map& by_id, + absl::flat_hash_map& by_signature, + Overload&& overload, absl::Status& status) { if (!status.ok()) { return; } - if (overload.id().empty()) { - OverloadDecl overload_decl = overload; - absl::StatusOr overload_id = - common_internal::MakeOverloadSignature( - function_name, overload_decl.args(), overload_decl.member()); - if (!overload_id.ok()) { - status = overload_id.status(); - return; - } - overload_decl.set_id(*overload_id); - AddOverloadInternal(function_name, insertion_order, overloads, - std::move(overload_decl), status); + absl::StatusOr signature = + common_internal::MakeOverloadSignature(function_name, overload.args(), + overload.member()); + if (!signature.ok()) { + status = signature.status(); return; } - if (auto it = overloads.find(overload.id()); it != overloads.end()) { + OverloadDecl mutable_overload = std::forward(overload); + + if (mutable_overload.id().empty()) { + mutable_overload.set_id(*signature); + } + + if (auto it = by_id.find(mutable_overload.id()); it != by_id.end()) { status = absl::AlreadyExistsError( - absl::StrCat("overload already exists: ", overload.id())); + absl::StrCat("overload exists: ", mutable_overload.id())); return; } - for (const auto& existing : overloads) { - if (SignaturesOverlap(overload, existing)) { + + for (const auto& existing : insertion_order) { + if (SignaturesOverlap(mutable_overload, existing)) { status = absl::InvalidArgumentError( absl::StrCat("overload signature collision: ", existing.id(), - " collides with ", overload.id())); + " collides with ", mutable_overload.id())); return; } } - const auto inserted = overloads.insert(std::forward(overload)); - ABSL_DCHECK(inserted.second); - insertion_order.push_back(*inserted.first); + + size_t index = insertion_order.size(); + by_id[mutable_overload.id()] = index; + by_signature[*signature] = index; + insertion_order.push_back(std::move(mutable_overload)); } void CollectTypeParams(absl::flat_hash_set& type_params, @@ -195,14 +198,25 @@ absl::flat_hash_set OverloadDecl::GetTypeParams() const { void FunctionDecl::AddOverloadImpl(const OverloadDecl& overload, absl::Status& status) { - AddOverloadInternal(name_, overloads_.insertion_order, overloads_.set, - overload, status); + AddOverloadInternal(name_, overloads_.insertion_order, overloads_.by_id, + overloads_.by_signature, overload, status); } void FunctionDecl::AddOverloadImpl(OverloadDecl&& overload, absl::Status& status) { - AddOverloadInternal(name_, overloads_.insertion_order, overloads_.set, - std::move(overload), status); + AddOverloadInternal(name_, overloads_.insertion_order, overloads_.by_id, + overloads_.by_signature, std::move(overload), status); +} + +const OverloadDecl* FunctionDecl::FindOverloadById(absl::string_view id) const { + if (auto it = overloads_.by_id.find(id); it != overloads_.by_id.end()) { + return &overloads_.insertion_order[it->second]; + } + if (auto it = overloads_.by_signature.find(id); + it != overloads_.by_signature.end()) { + return &overloads_.insertion_order[it->second]; + } + return nullptr; } } // namespace cel diff --git a/common/decl.h b/common/decl.h index 22ee8cbf0..7aea8c502 100644 --- a/common/decl.h +++ b/common/decl.h @@ -22,11 +22,10 @@ #include "absl/algorithm/container.h" #include "absl/base/attributes.h" +#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" -#include "absl/hash/hash.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" @@ -264,39 +263,6 @@ OverloadDecl MakeMemberOverloadDecl(absl::string_view id, Type result, return overload_decl; } -struct OverloadDeclHash { - using is_transparent = void; - - size_t operator()(const OverloadDecl& overload_decl) const { - return (*this)(overload_decl.id()); - } - - size_t operator()(absl::string_view id) const { return absl::HashOf(id); } -}; - -struct OverloadDeclEqualTo { - using is_transparent = void; - - bool operator()(const OverloadDecl& lhs, const OverloadDecl& rhs) const { - return (*this)(lhs.id(), rhs.id()); - } - - bool operator()(const OverloadDecl& lhs, absl::string_view rhs) const { - return (*this)(lhs.id(), rhs); - } - - bool operator()(absl::string_view lhs, const OverloadDecl& rhs) const { - return (*this)(lhs, rhs.id()); - } - - bool operator()(absl::string_view lhs, absl::string_view rhs) const { - return lhs == rhs; - } -}; - -using OverloadDeclHashSet = - absl::flat_hash_set; - template absl::StatusOr MakeFunctionDecl(std::string name, Overloads&&... overloads); @@ -346,21 +312,27 @@ class FunctionDecl final { return overloads_.insertion_order; } + ABSL_MUST_USE_RESULT const OverloadDecl* FindOverloadById( + absl::string_view id) const; + std::vector release_overloads() { std::vector released = std::move(overloads_.insertion_order); overloads_.insertion_order.clear(); - overloads_.set.clear(); + overloads_.by_id.clear(); + overloads_.by_signature.clear(); return released; } private: struct Overloads { std::vector insertion_order; - OverloadDeclHashSet set; + absl::flat_hash_map by_id; + absl::flat_hash_map by_signature; void Reserve(size_t size) { insertion_order.reserve(size); - set.reserve(size); + by_id.reserve(size); + by_signature.reserve(size); } }; diff --git a/common/decl_test.cc b/common/decl_test.cc index 510cd5017..72e7f1b93 100644 --- a/common/decl_test.cc +++ b/common/decl_test.cc @@ -165,6 +165,53 @@ TEST(FunctionDecl, Overloads) { StatusIs(absl::StatusCode::kInvalidArgument)); } +TEST(FunctionDecl, AddOverloadInvalidSignature) { + FunctionDecl function_decl; + function_decl.set_name("foo"); + // Member overload must have at least one argument (the receiver). + // This should fail to add because signature generation fails. + EXPECT_THAT(function_decl.AddOverload(MakeMemberOverloadDecl(StringType{})), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST(FunctionDecl, AddOverloadDuplicateId) { + ASSERT_OK_AND_ASSIGN( + auto function_decl, + MakeFunctionDecl("hello", + MakeOverloadDecl("foo", StringType{}, StringType{}))); + // Adding another overload with the same ID "foo" should fail. + EXPECT_THAT( + function_decl.AddOverload(MakeOverloadDecl("foo", IntType{}, IntType{})), + StatusIs(absl::StatusCode::kAlreadyExists)); +} + +TEST(FunctionDecl, FindOverload) { + ASSERT_OK_AND_ASSIGN( + auto function_decl, + MakeFunctionDecl( + "hello", MakeOverloadDecl("foo", StringType{}, StringType{}), + MakeMemberOverloadDecl("bar", StringType{}, StringType{}), + MakeOverloadDecl(IntType{}, IntType{}))); + + // Find by explicit ID + const OverloadDecl* overload = function_decl.FindOverloadById("foo"); + ASSERT_NE(overload, nullptr); + EXPECT_EQ(overload->id(), "foo"); + + // Find by ID fallback to signature + overload = function_decl.FindOverloadById("hello(string)"); + ASSERT_NE(overload, nullptr); + EXPECT_EQ(overload->id(), "foo"); + + // Find implicit overload (where ID == signature) + overload = function_decl.FindOverloadById("hello(int)"); + ASSERT_NE(overload, nullptr); + EXPECT_EQ(overload->id(), "hello(int)"); + + // Non-existent + EXPECT_EQ(function_decl.FindOverloadById("non_existent"), nullptr); +} + TEST(FunctionDecl, OverloadId) { google::protobuf::Arena arena; const auto* descriptor = diff --git a/env/BUILD b/env/BUILD index bd82e8ec6..1816238a5 100644 --- a/env/BUILD +++ b/env/BUILD @@ -56,6 +56,7 @@ cc_library( "//common:container", "//common:decl", "//common:type", + "//common/internal:signature", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", diff --git a/env/config_test.cc b/env/config_test.cc index df0d6f875..8cfc3cf7f 100644 --- a/env/config_test.cc +++ b/env/config_test.cc @@ -88,6 +88,34 @@ INSTANTIATE_TEST_SUITE_P( StandardLibraryConfigTestCase{ .standard_library_config = {}, }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .included_functions = {{"_+_", "add_int64"}, + {"_+_", "add_list"}}, + }, + }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .included_functions = {{"_+_", "add(int,int)"}, + {"_+_", "add(list,list)"}}, + }, + }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .excluded_functions = {{"_+_", "add_int64"}, + {"_+_", "add_list"}}, + }, + }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .excluded_functions = {{"_+_", "add(int,int)"}, + {"_+_", "add(list,list)"}}, + }, + }, StandardLibraryConfigTestCase{ .standard_library_config = { @@ -106,6 +134,15 @@ INSTANTIATE_TEST_SUITE_P( .expected_error = "Cannot set both included and excluded functions.", }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .included_functions = {{"_+_", "add(int,int)"}}, + .excluded_functions = {{"_-_", ""}}, + }, + .expected_error = + "Cannot set both included and excluded functions.", + }, StandardLibraryConfigTestCase{ .standard_library_config = { @@ -114,6 +151,15 @@ INSTANTIATE_TEST_SUITE_P( .expected_error = "Cannot include function '_+_' and also its " "specific overload 'add_list'", }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .included_functions = {{"_+_", ""}, + {"_+_", "add(int,int)"}}, + }, + .expected_error = "Cannot include function '_+_' and also its " + "specific overload 'add(int,int)'", + }, StandardLibraryConfigTestCase{ .standard_library_config = { @@ -121,6 +167,15 @@ INSTANTIATE_TEST_SUITE_P( }, .expected_error = "Cannot exclude function '_+_' and also its " "specific overload 'add_list'", + }, + StandardLibraryConfigTestCase{ + .standard_library_config = + { + .excluded_functions = {{"_+_", ""}, + {"_+_", "add(int,int)"}}, + }, + .expected_error = "Cannot exclude function '_+_' and also its " + "specific overload 'add(int,int)'", })); TEST(VariableConfigTest, VariableConfig) { diff --git a/env/env.cc b/env/env.cc index 22d24295e..4fa4e7398 100644 --- a/env/env.cc +++ b/env/env.cc @@ -26,6 +26,7 @@ #include "common/constant.h" #include "common/container.h" #include "common/decl.h" +#include "common/internal/signature.h" #include "common/type.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" @@ -57,21 +58,47 @@ bool ShouldIncludeMacro(const Config::StandardLibraryConfig& config, bool ShouldIncludeFunction(const Config::StandardLibraryConfig& config, absl::string_view function, - absl::string_view overload_id) { - if (config.excluded_functions.contains( - std::make_pair(std::string(function), std::string(overload_id))) || - config.excluded_functions.contains( - std::make_pair(std::string(function), ""))) { - return false; + const OverloadDecl& overload) { + if (config.excluded_functions.empty() && config.included_functions.empty()) { + return true; + } + + if (!config.excluded_functions.empty()) { + if (config.excluded_functions.contains(std::make_pair( + std::string(function), std::string(overload.id()))) || + config.excluded_functions.contains( + std::make_pair(std::string(function), ""))) { + return false; + } + absl::StatusOr signature = + common_internal::MakeOverloadSignature(function, overload.args(), + overload.member()); + if (signature.ok() && config.excluded_functions.contains(std::make_pair( + std::string(function), *std::move(signature)))) { + return false; + } } - if (!config.included_functions.empty() && - !config.included_functions.contains( - std::make_pair(std::string(function), "")) && - !config.included_functions.contains( - std::make_pair(std::string(function), std::string(overload_id)))) { + + if (!config.included_functions.empty()) { + if (config.included_functions.contains(std::make_pair( + std::string(function), std::string(overload.id()))) || + config.included_functions.contains( + std::make_pair(std::string(function), ""))) { + return true; + } + // Ok to call MakeOverloadSignature() again, because in practice either + // included or excluded functions may be specified, but not both. + absl::StatusOr signature = + common_internal::MakeOverloadSignature(function, overload.args(), + overload.member()); + if (signature.ok() && config.included_functions.contains(std::make_pair( + std::string(function), *std::move(signature)))) { + return true; + } return false; } - return true; + + return true; // Never reached } absl::StatusOr MakeStdlibSubset( @@ -87,9 +114,8 @@ absl::StatusOr MakeStdlibSubset( }; subset.should_include_overload = [&standard_library_config]( absl::string_view function, - absl::string_view overload_id) { - return ShouldIncludeFunction(standard_library_config, function, - overload_id); + const OverloadDecl& overload) { + return ShouldIncludeFunction(standard_library_config, function, overload); }; return subset; } diff --git a/env/env_test.cc b/env/env_test.cc index fda87dfab..00143a857 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -280,6 +280,15 @@ INSTANTIATE_TEST_SUITE_P( .expected_invalid_expressions = {"1 + 2", "[1, 2, 3] + [4, 5, 6]", "'hello' + 'world'"}, }, + StandardLibraryConfigTestCase{ + .standard_library_config = + {.excluded_functions = {{"_+_", "_+_(bytes,bytes)"}, + {"_+_", "_+_(list<~A>,list<~A>)"}, + {"_+_", "_+_(string,string)"}}}, + .expected_valid_expressions = {"1 + 2"}, + .expected_invalid_expressions = {"[1, 2, 3] + [4, 5, 6]", + "'hello' + 'world'"}, + }, StandardLibraryConfigTestCase{ .standard_library_config = {.excluded_functions = {{"_+_", "add_bytes"}, @@ -294,6 +303,13 @@ INSTANTIATE_TEST_SUITE_P( .expected_valid_expressions = {"1 + 2", "[1, 2, 3] + [4, 5, 6]", "'hello' + 'world'"}, }, + StandardLibraryConfigTestCase{ + .standard_library_config = + {.included_functions = {{"_+_", "_+_(int,int)"}, + {"_+_", "_+_(list<~A>,list<~A>)"}}}, + .expected_valid_expressions = {"1 + 2", "[1, 2, 3] + [4, 5, 6]"}, + .expected_invalid_expressions = {"'hello' + 'world'"}, + }, StandardLibraryConfigTestCase{ .standard_library_config = {.included_functions = {{"_+_", "add_int64"}, diff --git a/env/env_yaml_test.cc b/env/env_yaml_test.cc index 9c5b3f04f..38f08e371 100644 --- a/env/env_yaml_test.cc +++ b/env/env_yaml_test.cc @@ -153,7 +153,7 @@ TEST(EnvYamlTest, ParseStdlibConfig_InclusionStyle) { - name: "_+_" overloads: - id: add_bytes - - id: add_list + - id: "_+_(list<~A>,list<~A>)" - name: "matches" - name: "timestamp" overloads: @@ -166,7 +166,7 @@ TEST(EnvYamlTest, ParseStdlibConfig_InclusionStyle) { EXPECT_THAT( stdlib_config.included_functions, UnorderedElementsAre(std::make_pair("_+_", "add_bytes"), - std::make_pair("_+_", "add_list"), + std::make_pair("_+_", "_+_(list<~A>,list<~A>)"), std::make_pair("matches", ""), std::make_pair("timestamp", "string_to_timestamp"))) << " Actual stdlib config: " << stdlib_config; @@ -1405,9 +1405,9 @@ std::vector GetExportTestCases() { .included_functions = { std::make_pair("timestamp", "string_to_timestamp"), - std::make_pair("_+_", "add_list"), + std::make_pair("_+_", "_+_(list<~A>,list<~A>)"), std::make_pair("matches", ""), - std::make_pair("_+_", "add_bytes"), + std::make_pair("_+_", "_+_(bytes,bytes)"), }, })); return config; @@ -1417,8 +1417,8 @@ std::vector GetExportTestCases() { include_functions: - name: "_+_" overloads: - - id: "add_bytes" - - id: "add_list" + - id: "_+_(bytes,bytes)" + - id: "_+_(list<~A>,list<~A>)" - name: "matches" - name: "timestamp" overloads: From 903210f28babe29c18710d89217c0ce468fef9ba Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 16 Jun 2026 13:19:17 -0700 Subject: [PATCH 068/350] Internal change PiperOrigin-RevId: 933263369 --- checker/BUILD | 1 + checker/validation_result.h | 1 + common/decl.h | 64 +++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/checker/BUILD b/checker/BUILD index 10ed6e363..efca3ff73 100644 --- a/checker/BUILD +++ b/checker/BUILD @@ -49,6 +49,7 @@ cc_library( deps = [ ":type_check_issue", "//common:ast", + "//common:decl", "//common:source", "//common:type", "@com_google_absl//absl/base:nullability", diff --git a/checker/validation_result.h b/checker/validation_result.h index f424e7f6f..7417e9969 100644 --- a/checker/validation_result.h +++ b/checker/validation_result.h @@ -28,6 +28,7 @@ #include "absl/types/span.h" #include "checker/type_check_issue.h" #include "common/ast.h" +#include "common/decl.h" #include "common/source.h" #include "common/type.h" diff --git a/common/decl.h b/common/decl.h index 7aea8c502..b15645236 100644 --- a/common/decl.h +++ b/common/decl.h @@ -377,6 +377,70 @@ bool TypeIsAssignable(const Type& to, const Type& from); } // namespace common_internal +struct VariableDeclEqualTo { + using is_transparent = void; + + bool operator()(const cel::VariableDecl& lhs, + const cel::VariableDecl& rhs) const { + return lhs.name() == rhs.name(); + } + + bool operator()(const cel::VariableDecl& lhs, std::string_view rhs) const { + return lhs.name() == rhs; + } + + bool operator()(std::string_view lhs, const cel::VariableDecl& rhs) const { + return lhs == rhs.name(); + } +}; + +struct VariableDeclHash { + using is_transparent = void; + + size_t operator()(const cel::VariableDecl& decl) const { + return (*this)(decl.name()); + } + + size_t operator()(std::string_view name) const { return absl::HashOf(name); } +}; + +using VariableDeclSet = absl::flat_hash_set; + +struct FunctionDeclEqualTo { + using is_transparent = void; + + bool operator()(const cel::FunctionDecl& lhs, + const cel::FunctionDecl& rhs) const { + return (*this)(lhs.name(), rhs.name()); + } + + bool operator()(const cel::FunctionDecl& lhs, std::string_view rhs) const { + return (*this)(lhs.name(), rhs); + } + + bool operator()(std::string_view lhs, const cel::FunctionDecl& rhs) const { + return (*this)(lhs, rhs.name()); + } + + bool operator()(std::string_view lhs, std::string_view rhs) const { + return lhs == rhs; + } +}; + +struct FunctionDeclHash { + using is_transparent = void; + + size_t operator()(const cel::FunctionDecl& decl) const { + return absl::HashOf(decl.name()); + } + + size_t operator()(std::string_view name) const { return absl::HashOf(name); } +}; + +using FunctionDeclSet = absl::flat_hash_set; + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_DECL_H_ From 19b54d719884e57b5e41be5c79001d0a13c22c3c Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 16 Jun 2026 15:59:45 -0700 Subject: [PATCH 069/350] Add policy conformance tests to postsubmit windows bazel test workflow. PiperOrigin-RevId: 933348031 --- .github/workflows/windows_bazel_test.yml | 2 +- .github/workflows/windows_bazel_test_post_merge.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_bazel_test.yml b/.github/workflows/windows_bazel_test.yml index 4ac7f2eec..6d12e6861 100644 --- a/.github/workflows/windows_bazel_test.yml +++ b/.github/workflows/windows_bazel_test.yml @@ -25,4 +25,4 @@ jobs: # //... won't work. shell: bash run: | - bazelisk test --config=msvc conformance:all \ No newline at end of file + bazelisk test --config=msvc conformance:all conformance/policy:all \ No newline at end of file diff --git a/.github/workflows/windows_bazel_test_post_merge.yml b/.github/workflows/windows_bazel_test_post_merge.yml index 11801011e..569177fcc 100644 --- a/.github/workflows/windows_bazel_test_post_merge.yml +++ b/.github/workflows/windows_bazel_test_post_merge.yml @@ -9,5 +9,5 @@ jobs: trigger-test: # This prevents the workflow from running automatically when someone # pushes to their fork. - if: github.repository == 'google/cel-cpp' + if: github.repository == 'cel-expr/cel-cpp' uses: ./.github/workflows/windows_bazel_test.yml \ No newline at end of file From 9485d85c8d73d31ed0d21b8b21397c8d74b90584 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 16 Jun 2026 16:36:08 -0700 Subject: [PATCH 070/350] Update from google/cel-spec to cel-expr/cel-spec (Part 2) PiperOrigin-RevId: 933365198 --- README.md | 2 +- eval/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 41b44388d..7c3c26be0 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,4 @@ parser, and type checker. Released under the [Apache License](LICENSE). -[1]: https://github.com/google/cel-spec +[1]: https://github.com/cel-expr/cel-spec diff --git a/eval/README.md b/eval/README.md index ee6fd0798..32fa4bda4 100644 --- a/eval/README.md +++ b/eval/README.md @@ -3,4 +3,4 @@ A C++ implementation of a [Common Expression Language][1] evaluator. -[1]: https://github.com/google/cel-spec +[1]: https://github.com/cel-expr/cel-spec From 7dc7eacfbccf56c4d74bcb6e0f4dc2e63768c6bf Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 17 Jun 2026 01:08:33 -0700 Subject: [PATCH 071/350] No public description PiperOrigin-RevId: 933560360 --- .../descriptor_pool_type_introspector.cc | 18 +++++++++--------- .../descriptor_pool_type_introspector_test.cc | 8 ++++---- checker/internal/type_check_env.cc | 10 +++++----- checker/internal/type_checker_builder_impl.cc | 2 +- .../internal/type_checker_builder_impl_test.cc | 4 ++-- checker/internal/type_checker_impl.cc | 6 +++--- checker/internal/type_inference_context.cc | 4 ++-- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/checker/internal/descriptor_pool_type_introspector.cc b/checker/internal/descriptor_pool_type_introspector.cc index da4f4430b..733e4a3cb 100644 --- a/checker/internal/descriptor_pool_type_introspector.cc +++ b/checker/internal/descriptor_pool_type_introspector.cc @@ -42,7 +42,7 @@ FindStructTypeFieldByNameDirectly( const google::protobuf::Descriptor* absl_nullable descriptor = descriptor_pool->FindMessageTypeByName(type); if (descriptor == nullptr) { - return absl::nullopt; + return std::nullopt; } const google::protobuf::FieldDescriptor* absl_nullable field = descriptor->FindFieldByName(name); @@ -54,7 +54,7 @@ FindStructTypeFieldByNameDirectly( if (field != nullptr) { return StructTypeField(MessageTypeField(field)); } - return absl::nullopt; + return std::nullopt; } // Standard implementation for listing fields. @@ -67,7 +67,7 @@ ListStructTypeFieldsDirectly( const google::protobuf::Descriptor* absl_nullable descriptor = descriptor_pool->FindMessageTypeByName(type); if (descriptor == nullptr) { - return absl::nullopt; + return std::nullopt; } std::vector extensions; @@ -100,7 +100,7 @@ DescriptorPoolTypeIntrospector::FindTypeImpl(absl::string_view name) const { if (enum_descriptor != nullptr) { return Type::Enum(enum_descriptor); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> @@ -112,7 +112,7 @@ DescriptorPoolTypeIntrospector::FindEnumConstantImpl( const google::protobuf::EnumValueDescriptor* absl_nullable enum_value_descriptor = enum_descriptor->FindValueByName(value); if (enum_value_descriptor == nullptr) { - return absl::nullopt; + return std::nullopt; } return EnumConstant{ .type = Type::Enum(enum_descriptor), @@ -121,7 +121,7 @@ DescriptorPoolTypeIntrospector::FindEnumConstantImpl( .number = enum_value_descriptor->number(), }; } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> @@ -134,7 +134,7 @@ DescriptorPoolTypeIntrospector::FindStructTypeFieldByNameImpl( const FieldTable* field_table = GetFieldTable(type); if (field_table == nullptr) { - return absl::nullopt; + return std::nullopt; } if (auto it = field_table->json_name_map.find(name); @@ -147,7 +147,7 @@ DescriptorPoolTypeIntrospector::FindStructTypeFieldByNameImpl( return field_table->fields[it->second].field; } - return absl::nullopt; + return std::nullopt; } absl::StatusOr< @@ -160,7 +160,7 @@ DescriptorPoolTypeIntrospector::ListFieldsForStructTypeImpl( const FieldTable* field_table = GetFieldTable(type); if (field_table == nullptr) { - return absl::nullopt; + return std::nullopt; } std::vector fields; fields.reserve(field_table->non_extensions.size()); diff --git a/checker/internal/descriptor_pool_type_introspector_test.cc b/checker/internal/descriptor_pool_type_introspector_test.cc index 456798744..db766b347 100644 --- a/checker/internal/descriptor_pool_type_introspector_test.cc +++ b/checker/internal/descriptor_pool_type_introspector_test.cc @@ -47,7 +47,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, FindType) { "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"), IsOkAndHolds(Optional(Property(&Type::IsEnum, true)))); EXPECT_THAT(introspector.FindType("non.existent.Type"), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST(DescriptorPoolTypeIntrospectorTest, FindEnumConstant) { @@ -84,7 +84,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, auto field = introspector.FindStructTypeFieldByName( "cel.expr.conformance.proto3.TestAllTypes", "singleInt64"); - EXPECT_THAT(field, IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(field, IsOkAndHolds(Eq(std::nullopt))); } TEST(DescriptorPoolTypeIntrospectorTest, FindExtension) { @@ -108,7 +108,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, FindStructTypeFieldByNameWithJsonOpt) { auto field = introspector.FindStructTypeFieldByName( "cel.expr.conformance.proto3.TestAllTypes", "single_int64"); - ASSERT_THAT(field, IsOkAndHolds(Eq(absl::nullopt))); + ASSERT_THAT(field, IsOkAndHolds(Eq(std::nullopt))); } TEST(DescriptorPoolTypeIntrospectorTest, @@ -168,7 +168,7 @@ TEST(DescriptorPoolTypeIntrospectorTest, ListFieldsForStructTypeNotFound) { internal::GetTestingDescriptorPool()); auto fields = introspector.ListFieldsForStructType( "cel.expr.conformance.proto3.SomeOtherType"); - EXPECT_THAT(fields, IsOkAndHolds(Eq(absl::nullopt))); + EXPECT_THAT(fields, IsOkAndHolds(Eq(std::nullopt))); } } // namespace diff --git a/checker/internal/type_check_env.cc b/checker/internal/type_check_env.cc index 47487220c..8dc83518d 100644 --- a/checker/internal/type_check_env.cc +++ b/checker/internal/type_check_env.cc @@ -58,7 +58,7 @@ absl::StatusOr> TypeCheckEnv::LookupTypeName( return type; } } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> TypeCheckEnv::LookupEnumConstant( @@ -75,7 +75,7 @@ absl::StatusOr> TypeCheckEnv::LookupEnumConstant( return decl; } } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> TypeCheckEnv::LookupTypeConstant( @@ -92,14 +92,14 @@ absl::StatusOr> TypeCheckEnv::LookupTypeConstant( return LookupEnumConstant(enum_name_candidate, value_name_candidate); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> TypeCheckEnv::LookupStructField( absl::string_view type_name, absl::string_view field_name) const { if (proto_type_mask_registry_ != nullptr && !proto_type_mask_registry_->FieldIsVisible(type_name, field_name)) { - return absl::nullopt; + return std::nullopt; } // Check the type providers in registration order. // Note: this doesn't allow for shadowing a type with a subset type of the @@ -113,7 +113,7 @@ absl::StatusOr> TypeCheckEnv::LookupStructField( return field; } } - return absl::nullopt; + return std::nullopt; } const VariableDecl* absl_nullable VariableScope::LookupLocalVariable( diff --git a/checker/internal/type_checker_builder_impl.cc b/checker/internal/type_checker_builder_impl.cc index f0332b999..4289fb528 100644 --- a/checker/internal/type_checker_builder_impl.cc +++ b/checker/internal/type_checker_builder_impl.cc @@ -198,7 +198,7 @@ std::optional FilterDecl(FunctionDecl decl, } } if (filtered.overloads().empty()) { - return absl::nullopt; + return std::nullopt; } filtered.set_name(std::move(name)); return filtered; diff --git a/checker/internal/type_checker_builder_impl_test.cc b/checker/internal/type_checker_builder_impl_test.cc index 913e704ee..fa7f80960 100644 --- a/checker/internal/type_checker_builder_impl_test.cc +++ b/checker/internal/type_checker_builder_impl_test.cc @@ -349,7 +349,7 @@ TEST(ContextDeclsTest, CustomStructNotSupported) { if (name == "com.example.MyStruct") { return common_internal::MakeBasicStructType("com.example.MyStruct"); } - return absl::nullopt; + return std::nullopt; } }; @@ -370,7 +370,7 @@ TEST(ContextDeclsWithProtoTypeMaskTest, CustomStructNotSupported) { if (name == "com.example.MyStruct") { return common_internal::MakeBasicStructType("com.example.MyStruct"); } - return absl::nullopt; + return std::nullopt; } }; diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index f3a06a28d..2bc05dbf7 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -1098,11 +1098,11 @@ std::optional ResolveVisitor::CheckFieldType(int64_t id, auto field_info = env_->LookupStructField(struct_type.name(), field); if (!field_info.ok()) { status_.Update(field_info.status()); - return absl::nullopt; + return std::nullopt; } if (!field_info->has_value()) { ReportUndefinedField(id, field, struct_type.name()); - return absl::nullopt; + return std::nullopt; } auto type = field_info->value().GetType(); if (type.kind() == TypeKind::kEnum) { @@ -1134,7 +1134,7 @@ std::optional ResolveVisitor::CheckFieldType(int64_t id, "expression of type '", FormatTypeName(inference_context_->FinalizeType(operand_type)), "' cannot be the operand of a select operation"))); - return absl::nullopt; + return std::nullopt; } void ResolveVisitor::ResolveSelectOperation(const Expr& expr, diff --git a/checker/internal/type_inference_context.cc b/checker/internal/type_inference_context.cc index 4681784af..4f738b804 100644 --- a/checker/internal/type_inference_context.cc +++ b/checker/internal/type_inference_context.cc @@ -149,7 +149,7 @@ std::optional WrapperToPrimitive(const Type& t) { case TypeKind::kUintWrapper: return UintType(); default: - return absl::nullopt; + return std::nullopt; } } @@ -579,7 +579,7 @@ TypeInferenceContext::ResolveOverload(const FunctionDecl& decl, } if (!result_type.has_value() || matching_overloads.empty()) { - return absl::nullopt; + return std::nullopt; } return OverloadResolution{ .result_type = FullySubstitute(*result_type, /*free_to_dyn=*/false), From 8b7068abb4062074a135491ad8357139287084f9 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 17 Jun 2026 05:56:44 -0700 Subject: [PATCH 072/350] No public description PiperOrigin-RevId: 933675942 --- internal/message_equality_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/message_equality_test.cc b/internal/message_equality_test.cc index 318138d9b..092edd71b 100644 --- a/internal/message_equality_test.cc +++ b/internal/message_equality_test.cc @@ -399,7 +399,7 @@ absl::optional, PackTestAllTypesProto3Field(const google::protobuf::Message& message, const google::protobuf::FieldDescriptor* absl_nonnull field) { if (field->is_map()) { - return absl::nullopt; + return std::nullopt; } if (field->is_repeated() && field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE) { @@ -425,7 +425,7 @@ PackTestAllTypesProto3Field(const google::protobuf::Message& message, cel::to_address(packed), any_field)); return std::pair{packed, any_field}; } - return absl::nullopt; + return std::nullopt; } TEST_P(UnaryMessageFieldEqualsTest, Equals) { From 071fca088461260c8667b075d2c3c2a78d7b1fd8 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 17 Jun 2026 11:03:40 -0700 Subject: [PATCH 073/350] Add -fexceptions to the antlr gencode to avoid breaking users building with -fno-exceptions PiperOrigin-RevId: 933819478 --- bazel/antlr.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/antlr.bzl b/bazel/antlr.bzl index 2abbb6dbd..a4d28cdf8 100644 --- a/bazel/antlr.bzl +++ b/bazel/antlr.bzl @@ -55,6 +55,7 @@ def antlr_cc_library(name, src, package): generated, "@antlr4-cpp-runtime//:antlr4-cpp-runtime", ], + copts = ["-fexceptions"], linkstatic = 1, ) From e2ed5270e50f90a15e1008a18662cefbac303650 Mon Sep 17 00:00:00 2001 From: Justin King Date: Wed, 17 Jun 2026 14:08:48 -0700 Subject: [PATCH 074/350] Internal change PiperOrigin-RevId: 933914348 --- checker/internal/type_checker_impl.cc | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 2bc05dbf7..9c2806e89 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -199,6 +199,7 @@ class ResolveVisitor : public AstVisitorBase { struct AttributeResolution { const VariableDecl* decl; bool requires_disambiguation; + bool local; }; ResolveVisitor(NamespaceGenerator namespace_generator, @@ -1001,7 +1002,7 @@ void ResolveVisitor::ResolveSimpleIdentifier(const Expr& expr, const VariableDecl* local_decl = LookupLocalIdentifier(name); if (local_decl != nullptr && !absl::StartsWith(name, ".")) { - attributes_[&expr] = {local_decl, false}; + attributes_[&expr] = {local_decl, false, /*local=*/true}; types_[&expr] = inference_context_->InstantiateTypeParams(local_decl->type()); return; @@ -1016,8 +1017,13 @@ void ResolveVisitor::ResolveSimpleIdentifier(const Expr& expr, }); if (decl != nullptr) { - attributes_[&expr] = {decl, - /* requires_disambiguation= */ local_decl != nullptr}; + attributes_[&expr] = { + decl, + /* requires_disambiguation= */ local_decl != nullptr, + // There is some oddity here, `.` prefixed idents should never be local. + // So LookupLocalIdentifier above should never return a valid decl. + // Perhaps this is a refactor holdover? + /*local=*/false}; types_[&expr] = inference_context_->InstantiateTypeParams(decl->type()); return; } @@ -1072,9 +1078,14 @@ void ResolveVisitor::ResolveQualifiedIdentifier( root = &root->select_expr().operand(); } - attributes_[root] = {decl, - /* requires_disambiguation= */ decl != local_decl && - local_decl != nullptr}; + attributes_[root] = { + decl, + /* requires_disambiguation= */ decl != local_decl && + local_decl != nullptr, + // There is some oddity here, `.` prefixed idents should never be local. + // So LookupLocalIdentifier above should never return a valid decl. + // Perhaps this is a refactor holdover? + /*local=*/local_decl == decl}; types_[root] = inference_context_->InstantiateTypeParams(decl->type()); // fix-up select operations that were deferred. From 95e4b10bcc0ee5d3bd99650dc2a95e8fa2fee99a Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 17 Jun 2026 21:27:58 -0700 Subject: [PATCH 075/350] No public description PiperOrigin-RevId: 934095048 --- codelab/network_functions.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/codelab/network_functions.cc b/codelab/network_functions.cc index 64f199cb3..6cc1505a9 100644 --- a/codelab/network_functions.cc +++ b/codelab/network_functions.cc @@ -368,7 +368,7 @@ std::optional NetworkAddressRep::Unwrap( auto opaque = value.AsOpaque(); if (!opaque.has_value() || opaque->GetTypeId() != cel::TypeId()) { - return absl::nullopt; + return std::nullopt; } // Note: safety depends on: @@ -385,10 +385,10 @@ std::optional NetworkAddressRep::Parse( char ipv6[16]; auto version = ParseAddressImpl(str, &ipv4, ipv6); if (!version.ok()) { - return absl::nullopt; + return std::nullopt; } if (*version != IpVersion::kIPv4) { - return absl::nullopt; + return std::nullopt; } NetworkAddressRep rep; rep.version_ = *version; @@ -422,7 +422,7 @@ std::optional NetworkAddressMatcher::Parse( int dash_pos = str.find('-'); if (dash_pos == absl::string_view::npos) { // TODO(uncreated-issue/86): CIDR style addr/prefix-length - return absl::nullopt; + return std::nullopt; } absl::string_view min_str = str.substr(0, dash_pos); absl::string_view max_str = str.substr(dash_pos + 1); @@ -431,23 +431,23 @@ std::optional NetworkAddressMatcher::Parse( NetworkRangev6 v6; auto min_parse = ParseAddressImpl(min_str, &v4.min_incl, v6.min_incl); if (!min_parse.ok()) { - return absl::nullopt; + return std::nullopt; } auto max_parse = ParseAddressImpl(max_str, &v4.max_incl, v6.max_incl); if (!max_parse.ok()) { - return absl::nullopt; + return std::nullopt; } if (*min_parse != *max_parse) { - return absl::nullopt; + return std::nullopt; } NetworkAddressMatcher rep; if (*min_parse == IpVersion::kIPv4) { if (v4.min_incl > v4.max_incl) { - return absl::nullopt; + return std::nullopt; } rep.ranges_v4_.push_back(v4); } else if (*min_parse == IpVersion::kIPv6) { - return absl::nullopt; + return std::nullopt; } return rep; From 51faf9f1f2889bc1bbc582755ba37ed1d105f393 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 17 Jun 2026 22:43:18 -0700 Subject: [PATCH 076/350] No public description PiperOrigin-RevId: 934123553 --- extensions/select_optimization.cc | 12 ++++++------ extensions/select_optimization_test.cc | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 42cad0f92..0cc64311a 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -158,11 +158,11 @@ std::optional GetSelectInstruction( absl::string_view field_name) { auto field_or = planner_context.type_reflector() .FindStructTypeFieldByName(runtime_type, field_name) - .value_or(absl::nullopt); + .value_or(std::nullopt); if (field_or.has_value()) { return SelectInstruction{field_or->number(), std::string(field_or->name())}; } - return absl::nullopt; + return std::nullopt; } absl::StatusOr SelectQualifierFromList(const ListExpr& list) { @@ -410,7 +410,7 @@ class RewriterImpl : public AstRewriterBase { std::optional rt_type = (checker_type.has_message_type()) ? GetRuntimeType(checker_type.message_type().type()) - : absl::nullopt; + : std::nullopt; if (rt_type.has_value() && (*rt_type).Is()) { const StructType& runtime_type = rt_type->GetStruct(); std::optional field_or = @@ -540,7 +540,7 @@ class RewriterImpl : public AstRewriterBase { std::optional GetRuntimeType(absl::string_view type_name) { return planner_context_.type_reflector().FindType(type_name).value_or( - absl::nullopt); + std::nullopt); } void SetProgressStatus(const absl::Status& status) { @@ -600,7 +600,7 @@ class OptimizedSelectImpl { absl::StatusOr> CheckForMarkedAttributes( ExecutionFrameBase& frame, const AttributeTrail& attribute_trail) { if (attribute_trail.empty()) { - return absl::nullopt; + return std::nullopt; } if (frame.unknown_processing_enabled() && @@ -624,7 +624,7 @@ absl::StatusOr> CheckForMarkedAttributes( attribute_trail.attribute()); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr OptimizedSelectImpl::ApplySelect( diff --git a/extensions/select_optimization_test.cc b/extensions/select_optimization_test.cc index 9d4024098..c14c4d461 100644 --- a/extensions/select_optimization_test.cc +++ b/extensions/select_optimization_test.cc @@ -257,7 +257,7 @@ class MockAccessApis : public LegacyTypeInfoApis, public LegacyTypeAccessApis { std::optional< google::api::expr::runtime::LegacyTypeInfoApis::FieldDescription> FindFieldByName(absl::string_view field_name) const override { - return absl::nullopt; + return std::nullopt; } MOCK_METHOD(absl::StatusOr, GetField, From 80fc78aca1945cdf2653e6d888c5ff15058892c4 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 18 Jun 2026 00:27:29 -0700 Subject: [PATCH 077/350] No public description PiperOrigin-RevId: 934169077 --- base/operators.cc | 32 ++++++++++++++++---------------- base/operators_test.cc | 32 ++++++++++++++++---------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/base/operators.cc b/base/operators.cc index 805acc5a1..b7df40b27 100644 --- a/base/operators.cc +++ b/base/operators.cc @@ -179,13 +179,13 @@ CEL_INTERNAL_TERNARY_OPERATORS_ENUM(CEL_TERNARY_OPERATOR) absl::optional Operator::FindByName(absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(operators_by_name.cbegin(), operators_by_name.cend(), input, OperatorDataNameComparer{}); if (it == operators_by_name.cend() || (*it)->name != input) { - return absl::nullopt; + return std::nullopt; } return Operator(*it); } @@ -193,13 +193,13 @@ absl::optional Operator::FindByName(absl::string_view input) { absl::optional Operator::FindByDisplayName(absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(operators_by_display_name.cbegin(), operators_by_display_name.cend(), input, OperatorDataDisplayNameComparer{}); if (it == operators_by_name.cend() || (*it)->display_name != input) { - return absl::nullopt; + return std::nullopt; } return Operator(*it); } @@ -208,13 +208,13 @@ absl::optional UnaryOperator::FindByName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(unary_operators_by_name.cbegin(), unary_operators_by_name.cend(), input, OperatorDataNameComparer{}); if (it == unary_operators_by_name.cend() || (*it)->name != input) { - return absl::nullopt; + return std::nullopt; } return UnaryOperator(*it); } @@ -223,14 +223,14 @@ absl::optional UnaryOperator::FindByDisplayName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(unary_operators_by_display_name.cbegin(), unary_operators_by_display_name.cend(), input, OperatorDataDisplayNameComparer{}); if (it == unary_operators_by_display_name.cend() || (*it)->display_name != input) { - return absl::nullopt; + return std::nullopt; } return UnaryOperator(*it); } @@ -239,13 +239,13 @@ absl::optional BinaryOperator::FindByName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(binary_operators_by_name.cbegin(), binary_operators_by_name.cend(), input, OperatorDataNameComparer{}); if (it == binary_operators_by_name.cend() || (*it)->name != input) { - return absl::nullopt; + return std::nullopt; } return BinaryOperator(*it); } @@ -254,14 +254,14 @@ absl::optional BinaryOperator::FindByDisplayName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(binary_operators_by_display_name.cbegin(), binary_operators_by_display_name.cend(), input, OperatorDataDisplayNameComparer{}); if (it == binary_operators_by_display_name.cend() || (*it)->display_name != input) { - return absl::nullopt; + return std::nullopt; } return BinaryOperator(*it); } @@ -270,13 +270,13 @@ absl::optional TernaryOperator::FindByName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(ternary_operators_by_name.cbegin(), ternary_operators_by_name.cend(), input, OperatorDataNameComparer{}); if (it == ternary_operators_by_name.cend() || (*it)->name != input) { - return absl::nullopt; + return std::nullopt; } return TernaryOperator(*it); } @@ -285,14 +285,14 @@ absl::optional TernaryOperator::FindByDisplayName( absl::string_view input) { absl::call_once(operators_once_flag, InitializeOperators); if (input.empty()) { - return absl::nullopt; + return std::nullopt; } auto it = std::lower_bound(ternary_operators_by_display_name.cbegin(), ternary_operators_by_display_name.cend(), input, OperatorDataDisplayNameComparer{}); if (it == ternary_operators_by_display_name.cend() || (*it)->display_name != input) { - return absl::nullopt; + return std::nullopt; } return TernaryOperator(*it); } diff --git a/base/operators_test.cc b/base/operators_test.cc index fdf95e7ae..6049f76c8 100644 --- a/base/operators_test.cc +++ b/base/operators_test.cc @@ -130,55 +130,55 @@ CEL_INTERNAL_TERNARY_OPERATORS_ENUM(CEL_TERNARY_OPERATOR) TEST(Operator, FindByName) { EXPECT_THAT(Operator::FindByName("@in"), Optional(Eq(Operator::In()))); EXPECT_THAT(Operator::FindByName("_in_"), Optional(Eq(Operator::OldIn()))); - EXPECT_THAT(Operator::FindByName("in"), Eq(absl::nullopt)); - EXPECT_THAT(Operator::FindByName(""), Eq(absl::nullopt)); + EXPECT_THAT(Operator::FindByName("in"), Eq(std::nullopt)); + EXPECT_THAT(Operator::FindByName(""), Eq(std::nullopt)); } TEST(Operator, FindByDisplayName) { EXPECT_THAT(Operator::FindByDisplayName("-"), Optional(Eq(Operator::Subtract()))); - EXPECT_THAT(Operator::FindByDisplayName("@in"), Eq(absl::nullopt)); - EXPECT_THAT(Operator::FindByDisplayName(""), Eq(absl::nullopt)); + EXPECT_THAT(Operator::FindByDisplayName("@in"), Eq(std::nullopt)); + EXPECT_THAT(Operator::FindByDisplayName(""), Eq(std::nullopt)); } TEST(UnaryOperator, FindByName) { EXPECT_THAT(UnaryOperator::FindByName("-_"), Optional(Eq(Operator::Negate()))); - EXPECT_THAT(UnaryOperator::FindByName("_-_"), Eq(absl::nullopt)); - EXPECT_THAT(UnaryOperator::FindByName(""), Eq(absl::nullopt)); + EXPECT_THAT(UnaryOperator::FindByName("_-_"), Eq(std::nullopt)); + EXPECT_THAT(UnaryOperator::FindByName(""), Eq(std::nullopt)); } TEST(UnaryOperator, FindByDisplayName) { EXPECT_THAT(UnaryOperator::FindByDisplayName("-"), Optional(Eq(Operator::Negate()))); - EXPECT_THAT(UnaryOperator::FindByDisplayName("&&"), Eq(absl::nullopt)); - EXPECT_THAT(UnaryOperator::FindByDisplayName(""), Eq(absl::nullopt)); + EXPECT_THAT(UnaryOperator::FindByDisplayName("&&"), Eq(std::nullopt)); + EXPECT_THAT(UnaryOperator::FindByDisplayName(""), Eq(std::nullopt)); } TEST(BinaryOperator, FindByName) { EXPECT_THAT(BinaryOperator::FindByName("_-_"), Optional(Eq(Operator::Subtract()))); - EXPECT_THAT(BinaryOperator::FindByName("-_"), Eq(absl::nullopt)); - EXPECT_THAT(BinaryOperator::FindByName(""), Eq(absl::nullopt)); + EXPECT_THAT(BinaryOperator::FindByName("-_"), Eq(std::nullopt)); + EXPECT_THAT(BinaryOperator::FindByName(""), Eq(std::nullopt)); } TEST(BinaryOperator, FindByDisplayName) { EXPECT_THAT(BinaryOperator::FindByDisplayName("-"), Optional(Eq(Operator::Subtract()))); - EXPECT_THAT(BinaryOperator::FindByDisplayName("!"), Eq(absl::nullopt)); - EXPECT_THAT(BinaryOperator::FindByDisplayName(""), Eq(absl::nullopt)); + EXPECT_THAT(BinaryOperator::FindByDisplayName("!"), Eq(std::nullopt)); + EXPECT_THAT(BinaryOperator::FindByDisplayName(""), Eq(std::nullopt)); } TEST(TernaryOperator, FindByName) { EXPECT_THAT(TernaryOperator::FindByName("_?_:_"), Optional(Eq(TernaryOperator::Conditional()))); - EXPECT_THAT(TernaryOperator::FindByName("-_"), Eq(absl::nullopt)); - EXPECT_THAT(TernaryOperator::FindByName(""), Eq(absl::nullopt)); + EXPECT_THAT(TernaryOperator::FindByName("-_"), Eq(std::nullopt)); + EXPECT_THAT(TernaryOperator::FindByName(""), Eq(std::nullopt)); } TEST(TernaryOperator, FindByDisplayName) { - EXPECT_THAT(TernaryOperator::FindByDisplayName(""), Eq(absl::nullopt)); - EXPECT_THAT(TernaryOperator::FindByDisplayName("!"), Eq(absl::nullopt)); + EXPECT_THAT(TernaryOperator::FindByDisplayName(""), Eq(std::nullopt)); + EXPECT_THAT(TernaryOperator::FindByDisplayName("!"), Eq(std::nullopt)); } TEST(Operator, SupportsAbslHash) { From 8515127e730d338066a0f39d13317b0b4cebc32a Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 18 Jun 2026 03:35:24 -0700 Subject: [PATCH 078/350] No public description PiperOrigin-RevId: 934248225 --- tools/flatbuffers_backed_impl.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/flatbuffers_backed_impl.cc b/tools/flatbuffers_backed_impl.cc index 10c0b1cb8..2ee226859 100644 --- a/tools/flatbuffers_backed_impl.cc +++ b/tools/flatbuffers_backed_impl.cc @@ -127,7 +127,7 @@ class ObjectStringIndexedMapImpl : public CelMap { arena_, **it, schema_, object_, arena_)); } } - return absl::nullopt; + return std::nullopt; } absl::StatusOr ListKeys() const override { return &keys_; } @@ -188,7 +188,7 @@ absl::optional FlatBuffersMapImpl::operator[]( } auto field = keys_.fields->LookupByKey(cel_key.StringOrDie().value().data()); if (field == nullptr) { - return absl::nullopt; + return std::nullopt; } switch (field->type()->base_type()) { case reflection::Byte: @@ -323,15 +323,15 @@ absl::optional FlatBuffersMapImpl::operator[]( } default: // Unsupported vector base types - return absl::nullopt; + return std::nullopt; } break; } default: // Unsupported types: enums, unions, arrays - return absl::nullopt; + return std::nullopt; } - return absl::nullopt; + return std::nullopt; } const CelMap* CreateFlatBuffersBackedObject(const uint8_t* flatbuf, From 3244f4a55e713555fc0f40ecfea5e38b5e3ad492 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 18 Jun 2026 03:35:53 -0700 Subject: [PATCH 079/350] No public description PiperOrigin-RevId: 934248421 --- testutil/test_macros.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testutil/test_macros.cc b/testutil/test_macros.cc index 672439dc5..19e9a4844 100644 --- a/testutil/test_macros.cc +++ b/testutil/test_macros.cc @@ -40,7 +40,7 @@ bool IsCelNamespace(const Expr& target) { std::optional CelBlockMacroExpander(MacroExprFactory& factory, Expr& target, absl::Span args) { if (!IsCelNamespace(target)) { - return absl::nullopt; + return std::nullopt; } Expr& bindings_arg = args[0]; if (!bindings_arg.has_list_expr()) { @@ -53,7 +53,7 @@ std::optional CelBlockMacroExpander(MacroExprFactory& factory, std::optional CelIndexMacroExpander(MacroExprFactory& factory, Expr& target, absl::Span args) { if (!IsCelNamespace(target)) { - return absl::nullopt; + return std::nullopt; } Expr& index_arg = args[0]; if (!index_arg.has_const_expr() || !index_arg.const_expr().has_int_value()) { @@ -72,7 +72,7 @@ std::optional CelIterVarMacroExpander(MacroExprFactory& factory, Expr& target, absl::Span args) { if (!IsCelNamespace(target)) { - return absl::nullopt; + return std::nullopt; } Expr& depth_arg = args[0]; if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || @@ -96,7 +96,7 @@ std::optional CelAccuVarMacroExpander(MacroExprFactory& factory, Expr& target, absl::Span args) { if (!IsCelNamespace(target)) { - return absl::nullopt; + return std::nullopt; } Expr& depth_arg = args[0]; if (!depth_arg.has_const_expr() || !depth_arg.const_expr().has_int_value() || From 3456ec81c08ec5ce3980c75b9522e917683e58ed Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 18 Jun 2026 05:34:35 -0700 Subject: [PATCH 080/350] No public description PiperOrigin-RevId: 934294071 --- extensions/protobuf/bind_proto_to_activation_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/protobuf/bind_proto_to_activation_test.cc b/extensions/protobuf/bind_proto_to_activation_test.cc index fd79508ac..680b4b353 100644 --- a/extensions/protobuf/bind_proto_to_activation_test.cc +++ b/extensions/protobuf/bind_proto_to_activation_test.cc @@ -76,10 +76,10 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationSkip) { EXPECT_THAT(activation.FindVariable("single_int32", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(activation.FindVariable("single_sint32", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(BindProtoToActivationTest, BindProtoToActivationDefault) { From 774340d632ae13dc74c64f2eade30d8209686089 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 18 Jun 2026 06:50:51 -0700 Subject: [PATCH 081/350] No public description PiperOrigin-RevId: 934323942 --- parser/macro_registry.cc | 4 ++-- parser/macro_registry_test.cc | 4 ++-- parser/parser.cc | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/parser/macro_registry.cc b/parser/macro_registry.cc index 3a816b10e..d36761e87 100644 --- a/parser/macro_registry.cc +++ b/parser/macro_registry.cc @@ -55,7 +55,7 @@ absl::optional MacroRegistry::FindMacro(absl::string_view name, bool receiver_style) const { // :: if (name.empty() || absl::StrContains(name, ':')) { - return absl::nullopt; + return std::nullopt; } // Try argument count specific key first. auto key = absl::StrCat(name, ":", arg_count, ":", @@ -68,7 +68,7 @@ absl::optional MacroRegistry::FindMacro(absl::string_view name, if (auto it = macros_.find(key); it != macros_.end()) { return it->second; } - return absl::nullopt; + return std::nullopt; } std::vector MacroRegistry::ListMacros() const { diff --git a/parser/macro_registry_test.cc b/parser/macro_registry_test.cc index 9e6da87a4..db8a99ab2 100644 --- a/parser/macro_registry_test.cc +++ b/parser/macro_registry_test.cc @@ -30,14 +30,14 @@ using ::testing::Ne; TEST(MacroRegistry, RegisterAndFind) { MacroRegistry macros; EXPECT_THAT(macros.RegisterMacro(HasMacro()), IsOk()); - EXPECT_THAT(macros.FindMacro("has", 1, false), Ne(absl::nullopt)); + EXPECT_THAT(macros.FindMacro("has", 1, false), Ne(std::nullopt)); } TEST(MacroRegistry, RegisterRollsback) { MacroRegistry macros; EXPECT_THAT(macros.RegisterMacros({HasMacro(), AllMacro(), AllMacro()}), StatusIs(absl::StatusCode::kAlreadyExists)); - EXPECT_THAT(macros.FindMacro("has", 1, false), Eq(absl::nullopt)); + EXPECT_THAT(macros.FindMacro("has", 1, false), Eq(std::nullopt)); } } // namespace diff --git a/parser/parser.cc b/parser/parser.cc index a858337a4..24b4ca079 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -1468,11 +1468,11 @@ Expr ParserVisitor::GlobalCallOrMacroImpl(int64_t expr_id, } } factory_.BeginMacro(factory_.GetSourceRange(expr_id)); - auto expr = macro->Expand(factory_, absl::nullopt, absl::MakeSpan(args)); + auto expr = macro->Expand(factory_, std::nullopt, absl::MakeSpan(args)); factory_.EndMacro(); if (expr) { if (add_macro_calls_) { - factory_.AddMacroCall(expr->id(), function, absl::nullopt, + factory_.AddMacroCall(expr->id(), function, std::nullopt, std::move(macro_args)); } // We did not end up using `expr_id`. Delete metadata. From 83ccadfe7e0d284956332f3b90804c23b6ef3caf Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 18 Jun 2026 15:33:26 -0700 Subject: [PATCH 082/350] Refactor local variable resolution in checker. No functional changes, but makes intent a little clearer w.r.t. checking if we need to preserve a leading '.' or not. PiperOrigin-RevId: 934579650 --- checker/internal/type_checker_impl.cc | 97 ++++++++++++++------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 9c2806e89..bca187417 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -960,11 +960,10 @@ void ResolveVisitor::ResolveFunctionOverloads(const Expr& expr, const VariableDecl* absl_nullable ResolveVisitor::LookupLocalIdentifier( absl::string_view name) { - // Note: if we see a leading dot, this shouldn't resolve to a local variable, - // but we need to check whether we need to disambiguate against a global in - // the reference map. if (absl::StartsWith(name, ".")) { - name = name.substr(1); + // Should not happen for normally parsed CEL, but prevent lookup in case + // of hand-crafted ASTs. + return nullptr; } return current_scope_->LookupLocalVariable(name); } @@ -999,13 +998,15 @@ void ResolveVisitor::ResolveSimpleIdentifier(const Expr& expr, absl::string_view name) { // Local variables (comprehension, bind) are simple identifiers so we can // skip generating the different namespace-qualified candidates. - const VariableDecl* local_decl = LookupLocalIdentifier(name); - - if (local_decl != nullptr && !absl::StartsWith(name, ".")) { - attributes_[&expr] = {local_decl, false, /*local=*/true}; - types_[&expr] = - inference_context_->InstantiateTypeParams(local_decl->type()); - return; + if (!absl::StartsWith(name, ".")) { + const VariableDecl* local_decl = LookupLocalIdentifier(name); + if (local_decl != nullptr) { + attributes_[&expr] = {local_decl, /*requires_disambiguation=*/false, + /*local=*/true}; + types_[&expr] = + inference_context_->InstantiateTypeParams(local_decl->type()); + return; + } } const VariableDecl* decl = nullptr; @@ -1016,14 +1017,13 @@ void ResolveVisitor::ResolveSimpleIdentifier(const Expr& expr, return decl == nullptr; }); + bool requires_disambiguation = false; + if (absl::StartsWith(name, ".")) { + requires_disambiguation = LookupLocalIdentifier(name.substr(1)) != nullptr; + } + if (decl != nullptr) { - attributes_[&expr] = { - decl, - /* requires_disambiguation= */ local_decl != nullptr, - // There is some oddity here, `.` prefixed idents should never be local. - // So LookupLocalIdentifier above should never return a valid decl. - // Perhaps this is a refactor holdover? - /*local=*/false}; + attributes_[&expr] = {decl, requires_disambiguation, /*local=*/false}; types_[&expr] = inference_context_->InstantiateTypeParams(decl->type()); return; } @@ -1039,35 +1039,49 @@ void ResolveVisitor::ResolveQualifiedIdentifier( return; } + int matched_segment_index = -1; + const VariableDecl* decl = nullptr; + bool requires_disambiguation = false; + bool is_local = false; // Local variables (comprehension, bind) are simple identifiers so we can // skip generating the different namespace-qualified candidates. - const VariableDecl* local_decl = LookupLocalIdentifier(qualifiers[0]); - const VariableDecl* decl = nullptr; - - int matched_segment_index = -1; - - if (local_decl != nullptr && !absl::StartsWith(qualifiers[0], ".")) { - decl = local_decl; - matched_segment_index = 0; - } else { - namespace_generator_.GenerateCandidates( - qualifiers, [&decl, &matched_segment_index, this]( - absl::string_view candidate, int segment_index) { - decl = LookupGlobalIdentifier(candidate); - if (decl != nullptr) { - matched_segment_index = segment_index; - return false; - } - return true; - }); + if (!absl::StartsWith(qualifiers[0], ".")) { + const VariableDecl* local_decl = LookupLocalIdentifier(qualifiers[0]); + if (local_decl != nullptr) { + decl = local_decl; + matched_segment_index = 0; + is_local = true; + goto resolve_select_trail; + } } + namespace_generator_.GenerateCandidates( + qualifiers, [&decl, &matched_segment_index, this]( + absl::string_view candidate, int segment_index) { + decl = LookupGlobalIdentifier(candidate); + if (decl != nullptr) { + matched_segment_index = segment_index; + return false; + } + return true; + }); + if (decl == nullptr) { ReportMissingReference(expr, FormatCandidate(qualifiers)); types_[&expr] = ErrorType(); return; } + if (absl::StartsWith(qualifiers[0], ".")) { + const VariableDecl* local_decl = + LookupLocalIdentifier(qualifiers[0].substr(1)); + if (local_decl != nullptr) { + requires_disambiguation = true; + } + } + +resolve_select_trail: + const int num_select_opts = qualifiers.size() - matched_segment_index - 1; const Expr* root = &expr; @@ -1078,14 +1092,7 @@ void ResolveVisitor::ResolveQualifiedIdentifier( root = &root->select_expr().operand(); } - attributes_[root] = { - decl, - /* requires_disambiguation= */ decl != local_decl && - local_decl != nullptr, - // There is some oddity here, `.` prefixed idents should never be local. - // So LookupLocalIdentifier above should never return a valid decl. - // Perhaps this is a refactor holdover? - /*local=*/local_decl == decl}; + attributes_[root] = {decl, requires_disambiguation, is_local}; types_[root] = inference_context_->InstantiateTypeParams(decl->type()); // fix-up select operations that were deferred. From ef57455367567055465d8d86032c54fb9db27aff Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 18 Jun 2026 19:41:49 -0700 Subject: [PATCH 083/350] No public description PiperOrigin-RevId: 934677950 --- extensions/bindings_ext.cc | 2 +- extensions/lists_functions.cc | 2 +- extensions/math_ext_macros.cc | 6 +++--- extensions/math_ext_test.cc | 4 ++-- extensions/proto_ext.cc | 12 ++++++------ 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/extensions/bindings_ext.cc b/extensions/bindings_ext.cc index c59f724bd..4823c077c 100644 --- a/extensions/bindings_ext.cc +++ b/extensions/bindings_ext.cc @@ -73,7 +73,7 @@ std::vector bindings_macros() { [](MacroExprFactory& factory, Expr& target, absl::Span args) -> absl::optional { if (!IsTargetNamespace(target)) { - return absl::nullopt; + return std::nullopt; } if (!args[0].has_ident_expr()) { return factory.ReportErrorAt( diff --git a/extensions/lists_functions.cc b/extensions/lists_functions.cc index 10bc717ed..bfe05d887 100644 --- a/extensions/lists_functions.cc +++ b/extensions/lists_functions.cc @@ -454,7 +454,7 @@ Macro ListSortByMacro() { MakeMapComprehension(factory, factory.Copy(sortby_input_ident), std::move(key_ident), std::move(key_expr)); if (!map_compr.has_value()) { - return absl::nullopt; + return std::nullopt; } // Build the call expression: diff --git a/extensions/math_ext_macros.cc b/extensions/math_ext_macros.cc index a66720a60..08b163132 100644 --- a/extensions/math_ext_macros.cc +++ b/extensions/math_ext_macros.cc @@ -72,7 +72,7 @@ absl::optional CheckInvalidArgs(MacroExprFactory &factory, } } - return absl::nullopt; + return std::nullopt; } bool IsListLiteralWithValidArgs(const Expr &arg) { @@ -99,7 +99,7 @@ std::vector math_macros() { [](MacroExprFactory &factory, Expr &target, absl::Span arguments) -> absl::optional { if (!IsTargetNamespace(target)) { - return absl::nullopt; + return std::nullopt; } switch (arguments.size()) { @@ -143,7 +143,7 @@ std::vector math_macros() { [](MacroExprFactory &factory, Expr &target, absl::Span arguments) -> absl::optional { if (!IsTargetNamespace(target)) { - return absl::nullopt; + return std::nullopt; } switch (arguments.size()) { diff --git a/extensions/math_ext_test.cc b/extensions/math_ext_test.cc index ea9331970..ce05ae6ed 100644 --- a/extensions/math_ext_test.cc +++ b/extensions/math_ext_test.cc @@ -93,7 +93,7 @@ TestCase MinCase(CelValue v1, CelValue v2, CelValue result) { } TestCase MinCase(CelValue list, CelValue result) { - return TestCase{kMathMin, list, absl::nullopt, result}; + return TestCase{kMathMin, list, std::nullopt, result}; } TestCase MaxCase(CelValue v1, CelValue v2, CelValue result) { @@ -101,7 +101,7 @@ TestCase MaxCase(CelValue v1, CelValue v2, CelValue result) { } TestCase MaxCase(CelValue list, CelValue result) { - return TestCase{kMathMax, list, absl::nullopt, result}; + return TestCase{kMathMax, list, std::nullopt, result}; } struct MacroTestCase { diff --git a/extensions/proto_ext.cc b/extensions/proto_ext.cc index f38039002..48618f7ae 100644 --- a/extensions/proto_ext.cc +++ b/extensions/proto_ext.cc @@ -45,11 +45,11 @@ absl::optional ValidateExtensionIdentifier(const Expr& expr) { absl::Overload( [](const SelectExpr& select_expr) -> absl::optional { if (select_expr.test_only()) { - return absl::nullopt; + return std::nullopt; } auto op_name = ValidateExtensionIdentifier(select_expr.operand()); if (!op_name.has_value()) { - return absl::nullopt; + return std::nullopt; } return absl::StrCat(*op_name, ".", select_expr.field()); }, @@ -57,7 +57,7 @@ absl::optional ValidateExtensionIdentifier(const Expr& expr) { return ident_expr.name(); }, [](const auto&) -> absl::optional { - return absl::nullopt; + return std::nullopt; }), expr.kind()); } @@ -68,7 +68,7 @@ absl::optional GetExtensionFieldName(const Expr& expr) { select_expr) { return ValidateExtensionIdentifier(expr); } - return absl::nullopt; + return std::nullopt; } bool IsExtensionCall(const Expr& target) { @@ -95,7 +95,7 @@ std::vector proto_macros() { [](MacroExprFactory& factory, Expr& target, absl::Span arguments) -> absl::optional { if (!IsExtensionCall(target)) { - return absl::nullopt; + return std::nullopt; } auto extFieldName = GetExtensionFieldName(arguments[1]); if (!extFieldName.has_value()) { @@ -109,7 +109,7 @@ std::vector proto_macros() { [](MacroExprFactory& factory, Expr& target, absl::Span arguments) -> absl::optional { if (!IsExtensionCall(target)) { - return absl::nullopt; + return std::nullopt; } auto extFieldName = GetExtensionFieldName(arguments[1]); if (!extFieldName.has_value()) { From 7641fac87020aae83f71cca82de1d7e759672b32 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Thu, 18 Jun 2026 23:58:02 -0700 Subject: [PATCH 084/350] No public description PiperOrigin-RevId: 934760042 --- eval/eval/attribute_utility.cc | 2 +- eval/eval/container_access_step.cc | 2 +- eval/eval/function_step.cc | 4 ++-- eval/eval/select_step.cc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eval/eval/attribute_utility.cc b/eval/eval/attribute_utility.cc index 117516caf..1e044627e 100644 --- a/eval/eval/attribute_utility.cc +++ b/eval/eval/attribute_utility.cc @@ -123,7 +123,7 @@ absl::optional AttributeUtility::MergeUnknowns( } if (!result_set.has_value()) { - return absl::nullopt; + return std::nullopt; } return UnknownValue(cel::Unknown(result_set->unknown_attributes(), diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index fda51e34f..4cf4ebf4d 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -55,7 +55,7 @@ absl::optional CelNumberFromValue(const Value& value) { case ValueKind::kDouble: return Number::FromDouble(value.GetDouble().NativeValue()); default: - return absl::nullopt; + return std::nullopt; } } diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index fcf429378..12c5af8a7 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -291,7 +291,7 @@ absl::StatusOr ResolveStatic( return overload; } } - return absl::nullopt; + return std::nullopt; } absl::StatusOr ResolveLazy( @@ -299,7 +299,7 @@ absl::StatusOr ResolveLazy( bool receiver_style, absl::Span providers, const ExecutionFrameBase& frame) { - ResolveResult result = absl::nullopt; + ResolveResult result = std::nullopt; std::vector arg_types(input_args.size()); diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 420f3ac31..b815f5d87 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -72,7 +72,7 @@ absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, return cel::ErrorValue(std::move(result).status()); } - return absl::nullopt; + return std::nullopt; } void TestOnlySelect(const StructValue& msg, const std::string& field, From aea9b2adcddea6595548b65ad4e8a70d9e4de04c Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Fri, 19 Jun 2026 03:50:29 -0700 Subject: [PATCH 085/350] No public description PiperOrigin-RevId: 934844207 --- eval/compiler/flat_expr_builder.cc | 2 +- eval/compiler/flat_expr_builder_extensions.cc | 6 +++--- eval/compiler/qualified_reference_resolver.cc | 10 +++++----- eval/compiler/regex_precompilation_optimization.cc | 4 ++-- eval/compiler/resolver.cc | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index fc6d87b16..aa9a8858c 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -1078,7 +1078,7 @@ class FlatExprVisitor : public cel::AstVisitor { // eligible for recursion, or nullopt if it is not. std::optional RecursionEligible() { if (!PlanRecursiveProgram() || program_builder_.current() == nullptr) { - return absl::nullopt; + return std::nullopt; } return program_builder_.current()->RecursiveDependencyDepth(); } diff --git a/eval/compiler/flat_expr_builder_extensions.cc b/eval/compiler/flat_expr_builder_extensions.cc index e51b64023..ee106ff4a 100644 --- a/eval/compiler/flat_expr_builder_extensions.cc +++ b/eval/compiler/flat_expr_builder_extensions.cc @@ -102,15 +102,15 @@ std::optional Subexpression::RecursiveDependencyDepth() const { auto* tree = absl::get_if(&program_); int depth = 0; if (tree == nullptr) { - return absl::nullopt; + return std::nullopt; } for (const auto& element : *tree) { auto* subexpression = absl::get_if(&element); if (subexpression == nullptr) { - return absl::nullopt; + return std::nullopt; } if (!(*subexpression)->IsRecursive()) { - return absl::nullopt; + return std::nullopt; } depth = std::max(depth, (*subexpression)->recursive_program().depth); } diff --git a/eval/compiler/qualified_reference_resolver.cc b/eval/compiler/qualified_reference_resolver.cc index 67c14d9b2..158e492be 100644 --- a/eval/compiler/qualified_reference_resolver.cc +++ b/eval/compiler/qualified_reference_resolver.cc @@ -99,7 +99,7 @@ std::optional BestOverloadMatch(const Resolver& resolver, return *name; } } - return absl::nullopt; + return std::nullopt; } // Rewriter visitor for resolving references. @@ -267,22 +267,22 @@ class ReferenceResolver : public cel::AstRewriterBase { if (rewritten_reference_.find(expr.id()) != rewritten_reference_.end()) { // The target expr matches a reference (resolved to an ident decl). // This should not be treated as a function qualifier. - return absl::nullopt; + return std::nullopt; } if (expr.has_ident_expr()) { return expr.ident_expr().name(); } else if (expr.has_select_expr()) { if (expr.select_expr().test_only()) { - return absl::nullopt; + return std::nullopt; } maybe_parent_namespace = ToNamespace(expr.select_expr().operand()); if (!maybe_parent_namespace.has_value()) { - return absl::nullopt; + return std::nullopt; } return absl::StrCat(*maybe_parent_namespace, ".", expr.select_expr().field()); } else { - return absl::nullopt; + return std::nullopt; } } diff --git a/eval/compiler/regex_precompilation_optimization.cc b/eval/compiler/regex_precompilation_optimization.cc index 455796131..38ef842b9 100644 --- a/eval/compiler/regex_precompilation_optimization.cc +++ b/eval/compiler/regex_precompilation_optimization.cc @@ -178,7 +178,7 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { if (subexpression == nullptr || subexpression->IsFlattened()) { // Already modified, can't recover the input pattern. - return absl::nullopt; + return std::nullopt; } std::optional constant; if (subexpression->IsRecursive()) { @@ -206,7 +206,7 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { return Cast(*constant).ToString(); } - return absl::nullopt; + return std::nullopt; } absl::Status RewritePlan( diff --git a/eval/compiler/resolver.cc b/eval/compiler/resolver.cc index 17f60eaad..cca72964a 100644 --- a/eval/compiler/resolver.cc +++ b/eval/compiler/resolver.cc @@ -128,7 +128,7 @@ std::optional Resolver::FindConstant(absl::string_view name, return TypeValue(**type_value); } } - return absl::nullopt; + return std::nullopt; } std::vector Resolver::FindOverloads( @@ -216,7 +216,7 @@ Resolver::FindType(absl::string_view name, int64_t expr_id) const { return std::make_pair(std::move(qualified_name), std::move(*maybe_type)); } } - return absl::nullopt; + return std::nullopt; } } // namespace google::api::expr::runtime From e54b8bd8322454b485852227512a6b458a26f4a8 Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 19 Jun 2026 11:29:23 -0700 Subject: [PATCH 086/350] Actually reject non-simple variable names in comprehensions PiperOrigin-RevId: 934996645 --- extensions/comprehensions_v2_macros.cc | 43 +++++++++++++++----------- parser/macro.cc | 22 ++++++++----- parser/parser_test.cc | 21 +++++++++++++ 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/extensions/comprehensions_v2_macros.cc b/extensions/comprehensions_v2_macros.cc index 134fb80ff..a054626f9 100644 --- a/extensions/comprehensions_v2_macros.cc +++ b/extensions/comprehensions_v2_macros.cc @@ -14,12 +14,14 @@ #include "extensions/comprehensions_v2_macros.h" +#include #include #include #include "absl/base/no_destructor.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/types/optional.h" #include "absl/types/span.h" @@ -38,16 +40,21 @@ namespace { using ::google::api::expr::common::CelOperator; +bool IsSimpleIdentifier(const Expr& expr) { + return expr.has_ident_expr() && !expr.ident_expr().name().empty() && + !absl::StartsWith(expr.ident_expr().name(), "."); +} + absl::optional ExpandAllMacro2(MacroExprFactory& factory, Expr& target, absl::Span args) { if (args.size() != 3) { return factory.ReportError("all() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "all() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "all() second variable name must be a simple identifier"); } @@ -89,11 +96,11 @@ absl::optional ExpandExistsMacro2(MacroExprFactory& factory, Expr& target, if (args.size() != 3) { return factory.ReportError("exists() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "exists() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "exists() second variable name must be a simple identifier"); } @@ -138,11 +145,11 @@ absl::optional ExpandExistsOneMacro2(MacroExprFactory& factory, if (args.size() != 3) { return factory.ReportError("existsOne() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "existsOne() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "existsOne() second variable name must be a simple identifier"); @@ -190,12 +197,12 @@ absl::optional ExpandTransformList3Macro(MacroExprFactory& factory, if (args.size() != 3) { return factory.ReportError("transformList() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformList() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformList() second variable name must be a simple identifier"); @@ -239,12 +246,12 @@ absl::optional ExpandTransformList4Macro(MacroExprFactory& factory, if (args.size() != 4) { return factory.ReportError("transformList() requires 4 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformList() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformList() second variable name must be a simple identifier"); @@ -290,12 +297,12 @@ absl::optional ExpandTransformMap3Macro(MacroExprFactory& factory, if (args.size() != 3) { return factory.ReportError("transformMap() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformMap() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformMap() second variable name must be a simple identifier"); @@ -338,12 +345,12 @@ absl::optional ExpandTransformMap4Macro(MacroExprFactory& factory, if (args.size() != 4) { return factory.ReportError("transformMap() requires 4 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformMap() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformMap() second variable name must be a simple identifier"); @@ -388,12 +395,12 @@ absl::optional ExpandTransformMapEntry3Macro(MacroExprFactory& factory, if (args.size() != 3) { return factory.ReportError("transformMapEntry() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformMapEntry() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformMapEntry() second variable name must be a simple identifier"); @@ -438,12 +445,12 @@ absl::optional ExpandTransformMapEntry4Macro(MacroExprFactory& factory, if (args.size() != 4) { return factory.ReportError("transformMapEntry() requires 4 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "transformMapEntry() first variable name must be a simple identifier"); } - if (!args[1].has_ident_expr() || args[1].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[1])) { return factory.ReportErrorAt( args[1], "transformMapEntry() second variable name must be a simple identifier"); diff --git a/parser/macro.cc b/parser/macro.cc index 8f8c9e596..815b07401 100644 --- a/parser/macro.cc +++ b/parser/macro.cc @@ -25,6 +25,7 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -40,6 +41,11 @@ namespace { using google::api::expr::common::CelOperator; +bool IsSimpleIdentifier(const Expr& expr) { + return expr.has_ident_expr() && !expr.ident_expr().name().empty() && + !absl::StartsWith(expr.ident_expr().name(), "."); +} + inline MacroExpander ToMacroExpander(GlobalMacroExpander expander) { ABSL_DCHECK(expander); return [expander = std::move(expander)]( @@ -87,7 +93,7 @@ absl::optional ExpandAllMacro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("all() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "all() variable name must be a simple identifier"); } @@ -119,7 +125,7 @@ absl::optional ExpandExistsMacro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("exists() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "exists() variable name must be a simple identifier"); } @@ -153,7 +159,7 @@ absl::optional ExpandExistsOneMacro(MacroExprFactory& factory, if (args.size() != 2) { return factory.ReportError("exists_one() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "exists_one() variable name must be a simple identifier"); } @@ -192,7 +198,7 @@ absl::optional ExpandMap2Macro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("map() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "map() variable name must be a simple identifier"); } @@ -225,7 +231,7 @@ absl::optional ExpandMap3Macro(MacroExprFactory& factory, Expr& target, if (args.size() != 3) { return factory.ReportError("map() requires 3 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "map() variable name must be a simple identifier"); } @@ -260,7 +266,7 @@ absl::optional ExpandFilterMacro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("filter() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "filter() variable name must be a simple identifier"); } @@ -298,7 +304,7 @@ absl::optional ExpandOptMapMacro(MacroExprFactory& factory, Expr& target, if (args.size() != 2) { return factory.ReportError("optMap() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "optMap() variable name must be a simple identifier"); } @@ -337,7 +343,7 @@ absl::optional ExpandOptFlatMapMacro(MacroExprFactory& factory, if (args.size() != 2) { return factory.ReportError("optFlatMap() requires 2 arguments"); } - if (!args[0].has_ident_expr() || args[0].ident_expr().name().empty()) { + if (!IsSimpleIdentifier(args[0])) { return factory.ReportErrorAt( args[0], "optFlatMap() variable name must be a simple identifier"); } diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 1add80f84..35f11b413 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -631,6 +631,27 @@ std::vector test_cases = { "ERROR: :1:7: all() variable name must be a simple identifier\n" " | 1.all(2, 3)\n" " | ......^"}, + {"[].all(.x, x)", "", + "ERROR: :1:9: all() variable name must be a simple identifier\n" + " | [].all(.x, x)\n" + " | ........^"}, + {"[].exists(.x, x)", "", + "ERROR: :1:12: exists() variable name must be a simple identifier\n" + " | [].exists(.x, x)\n" + " | ...........^"}, + {"[].exists_one(.x, x)", "", + "ERROR: :1:16: exists_one() variable name must be a simple " + "identifier\n" + " | [].exists_one(.x, x)\n" + " | ...............^"}, + {"[].map(.x, x, x)", "", + "ERROR: :1:9: map() variable name must be a simple identifier\n" + " | [].map(.x, x, x)\n" + " | ........^"}, + {"[].filter(.x, x)", "", + "ERROR: :1:12: filter() variable name must be a simple identifier\n" + " | [].filter(.x, x)\n" + " | ...........^"}, {"x[\"a\"].single_int32 == 23", "_==_(\n" " _[_](\n" From 16074dcc55fddb53294aacd34afddd3db4a3138a Mon Sep 17 00:00:00 2001 From: Clayton Knittel Date: Sun, 21 Jun 2026 00:27:48 -0700 Subject: [PATCH 087/350] No public description PiperOrigin-RevId: 935529310 --- eval/public/message_wrapper_test.cc | 3 +-- .../structs/proto_message_type_adapter_test.cc | 4 +++- eval/public/testing/matchers.cc | 4 +--- internal/proto_matchers.h | 13 ++++++------- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/eval/public/message_wrapper_test.cc b/eval/public/message_wrapper_test.cc index ff0e691ab..15e5e88da 100644 --- a/eval/public/message_wrapper_test.cc +++ b/eval/public/message_wrapper_test.cc @@ -18,7 +18,6 @@ #include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/testutil/test_message.pb.h" -#include "internal/casts.h" #include "internal/testing.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -60,7 +59,7 @@ TEST(MessageWrapperBuilder, Builder) { static_cast(&test_message)); auto mutable_message = - cel::internal::down_cast(builder.message_ptr()); + google::protobuf::DownCastMessage(builder.message_ptr()); mutable_message->set_int64_value(20); mutable_message->set_double_value(12.3); diff --git a/eval/public/structs/proto_message_type_adapter_test.cc b/eval/public/structs/proto_message_type_adapter_test.cc index 32608bc3f..270fd3ce1 100644 --- a/eval/public/structs/proto_message_type_adapter_test.cc +++ b/eval/public/structs/proto_message_type_adapter_test.cc @@ -36,6 +36,7 @@ #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" +#include "google/protobuf/message_lite.h" namespace google::api::expr::runtime { namespace { @@ -725,7 +726,8 @@ TEST(ProtoMesssageTypeAdapter, TypeInfoMutator) { ASSERT_OK_AND_ASSIGN(MessageWrapper::Builder builder, api->NewInstance(manager)); - EXPECT_NE(dynamic_cast(builder.message_ptr()), nullptr); + EXPECT_NE(google::protobuf::DynamicCastMessage(builder.message_ptr()), + nullptr); } TEST(ProtoMesssageTypeAdapter, TypeInfoAccesor) { diff --git a/eval/public/testing/matchers.cc b/eval/public/testing/matchers.cc index f79071fce..4f728c730 100644 --- a/eval/public/testing/matchers.cc +++ b/eval/public/testing/matchers.cc @@ -7,7 +7,6 @@ #include "absl/strings/string_view.h" #include "eval/public/cel_value.h" #include "eval/public/set_util.h" -#include "internal/casts.h" #include "internal/testing.h" #include "google/protobuf/message.h" @@ -76,8 +75,7 @@ class CelValueMatcherImpl CelValue::MessageWrapper arg; return v.GetValue(&arg) && arg.HasFullProto() && underlying_type_matcher_.Matches( - cel::internal::down_cast( - arg.message_ptr())); + google::protobuf::DownCastMessage(arg.message_ptr())); } void DescribeTo(std::ostream* os) const override { diff --git a/internal/proto_matchers.h b/internal/proto_matchers.h index 76d844036..02250634b 100644 --- a/internal/proto_matchers.h +++ b/internal/proto_matchers.h @@ -21,7 +21,6 @@ #include "absl/log/absl_check.h" #include "absl/memory/memory.h" -#include "internal/casts.h" #include "internal/testing.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -43,13 +42,13 @@ class TextProtoMatcher { bool MatchAndExplain(const google::protobuf::MessageLite& p, ::testing::MatchResultListener* listener) const { - return MatchAndExplain(cel::internal::down_cast(p), + return MatchAndExplain(google::protobuf::DownCastMessage(p), listener); } bool MatchAndExplain(const google::protobuf::MessageLite* p, ::testing::MatchResultListener* listener) const { - return MatchAndExplain(cel::internal::down_cast(p), + return MatchAndExplain(google::protobuf::DownCastMessage(p), listener); } @@ -58,7 +57,7 @@ class TextProtoMatcher { auto message = absl::WrapUnique(p.New()); ABSL_CHECK(google::protobuf::TextFormat::ParseFromString(expected_, message.get())); return google::protobuf::util::MessageDifferencer::Equals( - *message, cel::internal::down_cast(p)); + *message, google::protobuf::DownCastMessage(p)); } bool MatchAndExplain(const google::protobuf::Message* p, @@ -66,7 +65,7 @@ class TextProtoMatcher { auto message = absl::WrapUnique(p->New()); ABSL_CHECK(google::protobuf::TextFormat::ParseFromString(expected_, message.get())); return google::protobuf::util::MessageDifferencer::Equals( - *message, cel::internal::down_cast(*p)); + *message, google::protobuf::DownCastMessage(*p)); } inline void DescribeTo(::std::ostream* os) const { *os << expected_; } @@ -93,13 +92,13 @@ class ProtoMatcher { bool MatchAndExplain(const google::protobuf::MessageLite& p, ::testing::MatchResultListener* listener) const { - return MatchAndExplain(cel::internal::down_cast(p), + return MatchAndExplain(google::protobuf::DownCastMessage(p), listener); } bool MatchAndExplain(const google::protobuf::MessageLite* p, ::testing::MatchResultListener* listener) const { - return MatchAndExplain(cel::internal::down_cast(p), + return MatchAndExplain(google::protobuf::DownCastMessage(p), listener); } From a32e1418a8385c4b9c1ebe3b5072aaa8fbdf23ce Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 22 Jun 2026 11:42:04 -0700 Subject: [PATCH 088/350] When generating YAML from config, skip overload_id if identical to signature PiperOrigin-RevId: 936168430 --- env/env_yaml.cc | 21 +++++++++++++------ env/env_yaml_test.cc | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/env/env_yaml.cc b/env/env_yaml.cc index e7b8a7885..d5e3ad059 100644 --- a/env/env_yaml.cc +++ b/env/env_yaml.cc @@ -725,6 +725,9 @@ absl::StatusOr ParseFunctionOverloadConfig( function_name, "\"")); } overload_config.is_member_function = parsed_signature.is_member; + if (overload_config.overload_id.empty()) { + overload_config.overload_id = signature; + } if (!parsed_signature.signature_type.has_function()) { return absl::InternalError(absl::StrCat( "Function overload signature has no function type: ", signature)); @@ -1101,11 +1104,8 @@ void EmitFunctionOverloadConfig( const Config::FunctionOverloadConfig& overload_config, YAML::Emitter& out, const EnvConfigToYamlOptions& options) { out << YAML::BeginMap; - if (!overload_config.overload_id.empty()) { - out << YAML::Key << "id"; - out << YAML::Value << YAML::DoubleQuoted << overload_config.overload_id; - } bool signature_generated = false; + std::string signature_str; if (options.use_type_signatures) { bool param_type_spec_generated = true; std::vector params; @@ -1123,12 +1123,21 @@ void EmitFunctionOverloadConfig( common_internal::MakeOverloadSignature( function_name, params, overload_config.is_member_function); if (signature.ok()) { - out << YAML::Key << "signature"; - out << YAML::Value << YAML::DoubleQuoted << *signature; + signature_str = std::move(*signature); signature_generated = true; } } } + if (!overload_config.overload_id.empty()) { + if (!signature_generated || overload_config.overload_id != signature_str) { + out << YAML::Key << "id"; + out << YAML::Value << YAML::DoubleQuoted << overload_config.overload_id; + } + } + if (signature_generated) { + out << YAML::Key << "signature"; + out << YAML::Value << YAML::DoubleQuoted << signature_str; + } if (!signature_generated) { if (overload_config.is_member_function) { out << YAML::Key << "target" << YAML::Value; diff --git a/env/env_yaml_test.cc b/env/env_yaml_test.cc index 38f08e371..c5bd1b787 100644 --- a/env/env_yaml_test.cc +++ b/env/env_yaml_test.cc @@ -545,12 +545,15 @@ std::vector GetParseFunctionTestCases() { .overload_configs = { Config::FunctionOverloadConfig{ + .overload_id = + "google.protobuf.StringValue.isEmpty()", .examples = {"''.isEmpty() // true"}, .is_member_function = true, .parameters = {{.name = "string_wrapper"}}, .return_type = {.name = "bool"}, }, Config::FunctionOverloadConfig{ + .overload_id = "list<~T>.isEmpty()", .examples = {"[].isEmpty() // true", "[1].isEmpty() // false"}, .is_member_function = true, @@ -635,6 +638,7 @@ std::vector GetParseFunctionTestCases() { .overload_configs = { Config::FunctionOverloadConfig{ + .overload_id = "contains(list<~T>, ~T)", .examples = {"contains([1, 2, 3], 2) // true"}, .is_member_function = false, .parameters = @@ -1740,6 +1744,45 @@ std::vector GetExportTestCases() { - type_name: "int" )yaml", }, + ExportTestCase{ + .config = []() -> absl::StatusOr { + Config config; + CEL_RETURN_IF_ERROR(config.AddFunctionConfig( + {.name = "foo", + .overload_configs = { + {.overload_id = "timestamp.foo(A<~B>)", + .is_member_function = true, + .parameters = {{.name = "timestamp"}, + {.name = "A", + .params = {{.name = "B", + .is_type_param = true}}}}, + .return_type = {.name = "int"}}, + }})); + return config; + }(), + .expected_yaml = R"yaml( + functions: + - name: "foo" + overloads: + - signature: "timestamp.foo(A<~B>)" + return: "int" + )yaml", + .expected_alt_yaml = R"yaml( + functions: + - name: "foo" + overloads: + - id: "timestamp.foo(A<~B>)" + target: + type_name: "timestamp" + args: + - type_name: "A" + params: + - type_name: "B" + is_type_param: true + return: + type_name: "int" + )yaml", + }, }; }; @@ -1888,6 +1931,13 @@ std::vector GetSignatureRoundTripTestCases() { signature: "foo(timestamp,A<~B>)" return: "list" )yaml", + R"yaml( + functions: + - name: "foo" + overloads: + - signature: "timestamp.foo(A<~B>)" + return: "int" + )yaml", }; } From 76ae0b3c1768d93a10270f904101de338867bdb1 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 22 Jun 2026 12:33:00 -0700 Subject: [PATCH 089/350] Move signature.h and signature.cc from common/internal to common This is done to make it visible to cel/python PiperOrigin-RevId: 936194518 --- checker/BUILD | 2 +- checker/type_checker_subset_factory.cc | 10 +++---- common/BUILD | 38 ++++++++++++++++++++++++- common/decl.cc | 5 ++-- common/internal/BUILD | 36 ----------------------- common/{internal => }/signature.cc | 6 ++-- common/{internal => }/signature.h | 10 +++---- common/{internal => }/signature_test.cc | 33 ++++++++++----------- env/BUILD | 4 +-- env/env.cc | 8 ++---- env/env_yaml.cc | 25 +++++++--------- 11 files changed, 83 insertions(+), 94 deletions(-) rename common/{internal => }/signature.cc (99%) rename common/{internal => }/signature.h (93%) rename common/{internal => }/signature_test.cc (96%) diff --git a/checker/BUILD b/checker/BUILD index efca3ff73..7f3ccfef7 100644 --- a/checker/BUILD +++ b/checker/BUILD @@ -231,7 +231,7 @@ cc_library( deps = [ ":type_checker_builder", "//common:decl", - "//common/internal:signature", + "//common:signature", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:span", diff --git a/checker/type_checker_subset_factory.cc b/checker/type_checker_subset_factory.cc index e5335e220..1b146c5a5 100644 --- a/checker/type_checker_subset_factory.cc +++ b/checker/type_checker_subset_factory.cc @@ -22,7 +22,7 @@ #include "absl/types/span.h" #include "checker/type_checker_builder.h" #include "common/decl.h" -#include "common/internal/signature.h" +#include "common/signature.h" namespace cel { @@ -33,8 +33,8 @@ TypeCheckerSubset::FunctionPredicate IncludeOverloadsByIdPredicate( if (overload_ids.contains(overload.id())) { return true; } - auto signature = common_internal::MakeOverloadSignature( - function, overload.args(), overload.member()); + auto signature = + MakeOverloadSignature(function, overload.args(), overload.member()); return signature.ok() && overload_ids.contains(*signature); }; } @@ -52,8 +52,8 @@ TypeCheckerSubset::FunctionPredicate ExcludeOverloadsByIdPredicate( if (overload_ids.contains(overload.id())) { return false; } - auto signature = common_internal::MakeOverloadSignature( - function, overload.args(), overload.member()); + auto signature = + MakeOverloadSignature(function, overload.args(), overload.member()); return !signature.ok() || !overload_ids.contains(*signature); }; } diff --git a/common/BUILD b/common/BUILD index 0bd3632dd..0426c0827 100644 --- a/common/BUILD +++ b/common/BUILD @@ -79,6 +79,42 @@ cc_test( ], ) +cc_library( + name = "signature", + srcs = ["signature.cc"], + hdrs = ["signature.h"], + deps = [ + ":ast", + ":type", + ":type_spec_resolver", + "//internal:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "signature_test", + srcs = ["signature_test.cc"], + deps = [ + ":ast", + ":signature", + ":type", + ":type_kind", + ":type_spec_resolver", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "expr", srcs = ["expr.cc"], @@ -145,9 +181,9 @@ cc_library( hdrs = ["decl.h"], deps = [ ":constant", + ":signature", ":type", ":type_kind", - "//common/internal:signature", "//internal:status_macros", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:core_headers", diff --git a/common/decl.cc b/common/decl.cc index d2d50964a..858e6fb49 100644 --- a/common/decl.cc +++ b/common/decl.cc @@ -26,7 +26,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "common/internal/signature.h" +#include "common/signature.h" #include "common/type.h" #include "common/type_kind.h" @@ -117,8 +117,7 @@ void AddOverloadInternal(std::string_view function_name, } absl::StatusOr signature = - common_internal::MakeOverloadSignature(function_name, overload.args(), - overload.member()); + MakeOverloadSignature(function_name, overload.args(), overload.member()); if (!signature.ok()) { status = signature.status(); return; diff --git a/common/internal/BUILD b/common/internal/BUILD index b07faf229..3be350754 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -135,39 +135,3 @@ cc_library( "@com_google_protobuf//src/google/protobuf/io", ], ) - -cc_library( - name = "signature", - srcs = ["signature.cc"], - hdrs = ["signature.h"], - deps = [ - "//common:ast", - "//common:type", - "//common:type_spec_resolver", - "//internal:status_macros", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - "@com_google_protobuf//:protobuf", - ], -) - -cc_test( - name = "signature_test", - srcs = ["signature_test.cc"], - deps = [ - ":signature", - "//common:ast", - "//common:type", - "//common:type_kind", - "//common:type_spec_resolver", - "//internal:testing", - "//internal:testing_descriptor_pool", - "@com_google_absl//absl/base:no_destructor", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/status:statusor", - "@com_google_protobuf//:protobuf", - ], -) diff --git a/common/internal/signature.cc b/common/signature.cc similarity index 99% rename from common/internal/signature.cc rename to common/signature.cc index fe315bb04..e497e780d 100644 --- a/common/internal/signature.cc +++ b/common/signature.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "common/internal/signature.h" +#include "common/signature.h" #include #include @@ -34,7 +34,7 @@ #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" -namespace cel::common_internal { +namespace cel { // Signature generator helper functions. namespace { @@ -637,4 +637,4 @@ absl::StatusOr ParseType(std::string_view signature, google::protobuf::Are return cel::ConvertTypeSpecToType(type_spec, arena, pool); } -} // namespace cel::common_internal +} // namespace cel diff --git a/common/internal/signature.h b/common/signature.h similarity index 93% rename from common/internal/signature.h rename to common/signature.h index 8a44fbd5c..777f03439 100644 --- a/common/internal/signature.h +++ b/common/signature.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_SIGNATURE_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_SIGNATURE_H_ +#ifndef THIRD_PARTY_CEL_CPP_COMMON_SIGNATURE_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_SIGNATURE_H_ #include #include @@ -25,7 +25,7 @@ #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" -namespace cel::common_internal { +namespace cel { // Generates a signature for a `cel::Type`, which is a string representation of // the type. @@ -96,6 +96,6 @@ struct ParsedFunctionOverload { absl::StatusOr ParseFunctionSignature( std::string_view signature); -} // namespace cel::common_internal +} // namespace cel -#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_SIGNATURE_H_ +#endif // THIRD_PARTY_CEL_CPP_COMMON_SIGNATURE_H_ diff --git a/common/internal/signature_test.cc b/common/signature_test.cc similarity index 96% rename from common/internal/signature_test.cc rename to common/signature_test.cc index 17b628d88..ea51eb566 100644 --- a/common/internal/signature_test.cc +++ b/common/signature_test.cc @@ -1,4 +1,4 @@ -#include "common/internal/signature.h" +#include "common/signature.h" // Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,7 +30,7 @@ #include "internal/testing_descriptor_pool.h" #include "google/protobuf/arena.h" -namespace cel::common_internal { +namespace cel { namespace { using ::absl_testing::IsOkAndHolds; @@ -77,8 +77,7 @@ using TypeSignatureTest = testing::TestWithParam; TEST_P(TypeSignatureTest, TypeSignature) { const auto& param = GetParam(); - absl::StatusOr signature = - common_internal::MakeTypeSpecSignature(param.type); + absl::StatusOr signature = MakeTypeSpecSignature(param.type); if (!param.expected_error.empty()) { EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr(param.expected_error))); @@ -257,25 +256,24 @@ INSTANTIATE_TEST_SUITE_P(TypeSignatureTest, TypeSignatureTest, ValuesIn(GetTypeSignatureTestCases())); TEST(TypeSignatureTest, UnsupportedTypes) { - EXPECT_THAT(common_internal::MakeTypeSignature(UnknownType{}), + EXPECT_THAT(MakeTypeSignature(UnknownType{}), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Unsupported Type kind: *unknown*"))); - EXPECT_THAT(common_internal::MakeTypeSignature(ErrorType{}), + EXPECT_THAT(MakeTypeSignature(ErrorType{}), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Unsupported type in signature: *error*"))); - EXPECT_THAT(common_internal::MakeTypeSpecSignature( - TypeSpec(static_cast(999))), + EXPECT_THAT(MakeTypeSpecSignature(TypeSpec(static_cast(999))), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Unsupported primitive type"))); - EXPECT_THAT(common_internal::MakeTypeSpecSignature( - TypeSpec(static_cast(999))), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Unsupported well-known type"))); + EXPECT_THAT( + MakeTypeSpecSignature(TypeSpec(static_cast(999))), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unsupported well-known type"))); - EXPECT_THAT(common_internal::MakeTypeSpecSignature(TypeSpec( + EXPECT_THAT(MakeTypeSpecSignature(TypeSpec( PrimitiveTypeWrapper(static_cast(999)))), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Unsupported wrapper type"))); @@ -308,8 +306,7 @@ TEST_P(OverloadSignatureTest, OverloadSignature) { const auto& param = GetParam(); absl::StatusOr signature = - common_internal::MakeOverloadSignature(param.function_name, param.args, - param.is_member); + MakeOverloadSignature(param.function_name, param.args, param.is_member); if (!param.expected_error.empty()) { EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr(param.expected_error))); @@ -433,8 +430,8 @@ std::vector GetOverloadSignatureTestCases() { } TEST(OverloadSignatureTest, MemberFunctionNoReceiverError) { - auto signature = common_internal::MakeOverloadSignature( - "hello", std::vector{}, true); + auto signature = + MakeOverloadSignature("hello", std::vector{}, true); EXPECT_THAT(signature, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Member function with no receiver"))); @@ -784,4 +781,4 @@ TEST(OverloadSignatureTest, ArgumentTypeVector) { } } // namespace -} // namespace cel::common_internal +} // namespace cel diff --git a/env/BUILD b/env/BUILD index 1816238a5..0c17d6305 100644 --- a/env/BUILD +++ b/env/BUILD @@ -55,8 +55,8 @@ cc_library( "//common:constant", "//common:container", "//common:decl", + "//common:signature", "//common:type", - "//common/internal:signature", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", @@ -124,7 +124,7 @@ cc_library( ":config", "//common:ast", "//common:constant", - "//common/internal:signature", + "//common:signature", "//internal:status_macros", "//internal:strings", "@com_google_absl//absl/algorithm:container", diff --git a/env/env.cc b/env/env.cc index 4fa4e7398..85c5139da 100644 --- a/env/env.cc +++ b/env/env.cc @@ -26,7 +26,7 @@ #include "common/constant.h" #include "common/container.h" #include "common/decl.h" -#include "common/internal/signature.h" +#include "common/signature.h" #include "common/type.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" @@ -71,8 +71,7 @@ bool ShouldIncludeFunction(const Config::StandardLibraryConfig& config, return false; } absl::StatusOr signature = - common_internal::MakeOverloadSignature(function, overload.args(), - overload.member()); + MakeOverloadSignature(function, overload.args(), overload.member()); if (signature.ok() && config.excluded_functions.contains(std::make_pair( std::string(function), *std::move(signature)))) { return false; @@ -89,8 +88,7 @@ bool ShouldIncludeFunction(const Config::StandardLibraryConfig& config, // Ok to call MakeOverloadSignature() again, because in practice either // included or excluded functions may be specified, but not both. absl::StatusOr signature = - common_internal::MakeOverloadSignature(function, overload.args(), - overload.member()); + MakeOverloadSignature(function, overload.args(), overload.member()); if (signature.ok() && config.included_functions.contains(std::make_pair( std::string(function), *std::move(signature)))) { return true; diff --git a/env/env_yaml.cc b/env/env_yaml.cc index d5e3ad059..281cf3ff1 100644 --- a/env/env_yaml.cc +++ b/env/env_yaml.cc @@ -38,7 +38,7 @@ #include "absl/time/time.h" #include "common/ast.h" #include "common/constant.h" -#include "common/internal/signature.h" +#include "common/signature.h" #include "env/config.h" #include "env/type_info.h" #include "internal/status_macros.h" @@ -434,8 +434,7 @@ absl::StatusOr ParseTypeInfo(const YAML::Node& node, if (!type.IsScalar()) { return YamlError(yaml, type, "Node 'type' is not a string"); } - CEL_ASSIGN_OR_RETURN(auto type_spec, - common_internal::ParseTypeSpec(GetString(yaml, type))); + CEL_ASSIGN_OR_RETURN(auto type_spec, ParseTypeSpec(GetString(yaml, type))); CEL_ASSIGN_OR_RETURN(auto type_config, TypeSpecToTypeInfo(type_spec)); return type_config; } @@ -714,9 +713,8 @@ absl::StatusOr ParseFunctionOverloadConfig( } std::string signature = GetString(yaml, signature_node); - CEL_ASSIGN_OR_RETURN( - common_internal::ParsedFunctionOverload parsed_signature, - common_internal::ParseFunctionSignature(signature)); + CEL_ASSIGN_OR_RETURN(ParsedFunctionOverload parsed_signature, + ParseFunctionSignature(signature)); if (parsed_signature.function_name != function_name) { return YamlError(yaml, signature_node, absl::StrCat("Function overload name \"", @@ -767,8 +765,8 @@ absl::StatusOr ParseFunctionOverloadConfig( const YAML::Node return_type = overload["return"]; if (return_type.IsDefined()) { if (return_type.IsScalar()) { - CEL_ASSIGN_OR_RETURN(auto type_spec, common_internal::ParseTypeSpec( - GetString(yaml, return_type))); + CEL_ASSIGN_OR_RETURN(auto type_spec, + ParseTypeSpec(GetString(yaml, return_type))); CEL_ASSIGN_OR_RETURN(overload_config.return_type, TypeSpecToTypeInfo(type_spec)); } else if (return_type.IsMap()) { @@ -993,8 +991,7 @@ void EmitTypeInfo(const Config::TypeInfo& type_info, YAML::Emitter& out, if (options.use_type_signatures) { absl::StatusOr type_spec = TypeInfoToTypeSpec(type_info); if (type_spec.ok()) { - absl::StatusOr signature = - common_internal::MakeTypeSpecSignature(*type_spec); + absl::StatusOr signature = MakeTypeSpecSignature(*type_spec); if (signature.ok()) { out << YAML::Key << "type"; out << YAML::Value << YAML::DoubleQuoted << *signature; @@ -1119,9 +1116,8 @@ void EmitFunctionOverloadConfig( params.push_back(std::move(*type_spec)); } if (param_type_spec_generated) { - absl::StatusOr signature = - common_internal::MakeOverloadSignature( - function_name, params, overload_config.is_member_function); + absl::StatusOr signature = MakeOverloadSignature( + function_name, params, overload_config.is_member_function); if (signature.ok()) { signature_str = std::move(*signature); signature_generated = true; @@ -1177,8 +1173,7 @@ void EmitFunctionOverloadConfig( absl::StatusOr type_spec = TypeInfoToTypeSpec(overload_config.return_type); if (type_spec.ok()) { - absl::StatusOr signature = - common_internal::MakeTypeSpecSignature(*type_spec); + absl::StatusOr signature = MakeTypeSpecSignature(*type_spec); if (signature.ok()) { out << YAML::Key << "return"; out << YAML::Value << YAML::DoubleQuoted << *signature; From cba6300daa3b0578deaf83270229d1b018e79e2a Mon Sep 17 00:00:00 2001 From: sahvx655-wq Date: Tue, 23 Jun 2026 17:07:54 +0530 Subject: [PATCH 090/350] Add regression test for cord substring code unit length --- common/values/string_value_test.cc | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/common/values/string_value_test.cc b/common/values/string_value_test.cc index 201724905..380746e1e 100644 --- a/common/values/string_value_test.cc +++ b/common/values/string_value_test.cc @@ -412,6 +412,35 @@ TEST_F(StringValueTest, CharAt) { ".charAt(): is less than 0"))); } +TEST_F(StringValueTest, Substring) { + using ::cel::test::ErrorValueIs; + using ::cel::test::StringValueIs; + + // Each '€' is three bytes, so the cord built here is 18 bytes and is stored + // as a large (cord-backed) value. The substring length must be measured in + // code units, not code points, otherwise the cord overload truncates a + // multi-byte character or underflows the length. + StringValue unicode_cord = StringValue(absl::Cord("€€€€€€")); + StringValue unicode_view = StringValue("€€€€€€"); + + EXPECT_THAT(unicode_cord.Substring(0, 2), StringValueIs("€€")); + EXPECT_THAT(unicode_view.Substring(0, 2), StringValueIs("€€")); + EXPECT_THAT(unicode_cord.Substring(1, 2), StringValueIs("€")); + EXPECT_THAT(unicode_view.Substring(1, 2), StringValueIs("€")); + EXPECT_THAT(unicode_cord.Substring(2, 4), StringValueIs("€€")); + EXPECT_THAT(unicode_view.Substring(2, 4), StringValueIs("€€")); + EXPECT_THAT(unicode_cord.Substring(2), StringValueIs("€€€€")); + EXPECT_THAT(unicode_view.Substring(2), StringValueIs("€€€€")); + + EXPECT_THAT(unicode_cord.Substring(0, 7), + ErrorValueIs(absl::InvalidArgumentError( + ".substring(, ): or is " + "greater than .size()"))); + EXPECT_THAT(unicode_cord.Substring(-1), + ErrorValueIs(absl::InvalidArgumentError( + ".substring(): is less than 0"))); +} + TEST_F(StringValueTest, Join) { using ::cel::runtime_internal::CreateNoMatchingOverloadError; using ::cel::test::ErrorValueIs; From 452235ff20435bd4eaa753e62322cb09b0f365a9 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 23 Jun 2026 20:52:20 -0700 Subject: [PATCH 091/350] No public description PiperOrigin-RevId: 937065106 --- eval/public/structs/cel_proto_wrap_util.cc | 4 ++-- eval/public/structs/cel_proto_wrapper.cc | 2 +- eval/public/structs/legacy_type_provider.cc | 14 +++++++------- eval/public/structs/legacy_type_provider_test.cc | 14 +++++++------- eval/public/structs/proto_message_type_adapter.cc | 4 ++-- .../structs/proto_message_type_adapter_test.cc | 2 +- .../structs/protobuf_descriptor_type_provider.cc | 4 ++-- .../structs/trivial_legacy_type_info_test.cc | 4 ++-- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/eval/public/structs/cel_proto_wrap_util.cc b/eval/public/structs/cel_proto_wrap_util.cc index d0f80171f..7bfe81fe6 100644 --- a/eval/public/structs/cel_proto_wrap_util.cc +++ b/eval/public/structs/cel_proto_wrap_util.cc @@ -691,7 +691,7 @@ class ValueFromMessageMaker { return CreateWellknownTypeValue(message, factory, arena); // WELLKNOWNTYPE_FIELDMASK has no special CelValue type default: - return absl::nullopt; + return std::nullopt; } } @@ -716,7 +716,7 @@ absl::optional DynamicMap::operator[](CelValue key) const { auto it = values_->fields().find(std::string(str_key.value())); if (it == values_->fields().end()) { - return absl::nullopt; + return std::nullopt; } return ValueManager(factory_, arena_).ValueFromMessage(&it->second); diff --git a/eval/public/structs/cel_proto_wrapper.cc b/eval/public/structs/cel_proto_wrapper.cc index a1dc83ade..6fad6aee3 100644 --- a/eval/public/structs/cel_proto_wrapper.cc +++ b/eval/public/structs/cel_proto_wrapper.cc @@ -53,7 +53,7 @@ absl::optional CelProtoWrapper::MaybeWrapValue( if (msg != nullptr) { return InternalWrapMessage(msg); } else { - return absl::nullopt; + return std::nullopt; } } diff --git a/eval/public/structs/legacy_type_provider.cc b/eval/public/structs/legacy_type_provider.cc index f87ab9645..f8db92298 100644 --- a/eval/public/structs/legacy_type_provider.cc +++ b/eval/public/structs/legacy_type_provider.cc @@ -63,7 +63,7 @@ class LegacyStructValueBuilder final : public cel::StructValueBuilder { CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetField( name, legacy_value, memory_manager_, builder_)) .With(cel::ErrorValueReturn()); - return absl::nullopt; + return std::nullopt; } absl::StatusOr> SetFieldByNumber( @@ -76,7 +76,7 @@ class LegacyStructValueBuilder final : public cel::StructValueBuilder { CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetFieldByNumber( number, legacy_value, memory_manager_, builder_)) .With(cel::ErrorValueReturn()); - return absl::nullopt; + return std::nullopt; } absl::StatusOr Build() && override { @@ -116,7 +116,7 @@ class LegacyValueBuilder final : public cel::ValueBuilder { CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetField( name, legacy_value, memory_manager_, builder_)) .With(cel::ErrorValueReturn()); - return absl::nullopt; + return std::nullopt; } absl::StatusOr> SetFieldByNumber( @@ -129,7 +129,7 @@ class LegacyValueBuilder final : public cel::ValueBuilder { CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetFieldByNumber( number, legacy_value, memory_manager_, builder_)) .With(cel::ErrorValueReturn()); - return absl::nullopt; + return std::nullopt; } absl::StatusOr Build() && override { @@ -187,7 +187,7 @@ absl::StatusOr> LegacyTypeProvider::FindTypeImpl( return cel::common_internal::MakeBasicStructType( (*type_info)->GetTypename(MessageWrapper())); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> @@ -206,13 +206,13 @@ LegacyTypeProvider::FindStructTypeFieldByNameImpl( const auto* mutation_apis = (*type_info)->GetMutationApis(MessageWrapper()); if (mutation_apis == nullptr || !mutation_apis->DefinesField(name)) { - return absl::nullopt; + return std::nullopt; } return cel::common_internal::BasicStructTypeField(name, 0, cel::DynType{}); } } - return absl::nullopt; + return std::nullopt; } } // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_provider_test.cc b/eval/public/structs/legacy_type_provider_test.cc index 160ac49f3..8de2aba01 100644 --- a/eval/public/structs/legacy_type_provider_test.cc +++ b/eval/public/structs/legacy_type_provider_test.cc @@ -28,7 +28,7 @@ class LegacyTypeProviderTestEmpty : public LegacyTypeProvider { public: absl::optional ProvideLegacyType( absl::string_view name) const override { - return absl::nullopt; + return std::nullopt; } }; @@ -60,14 +60,14 @@ class LegacyTypeProviderTestImpl : public LegacyTypeProvider { if (name == "test") { return LegacyTypeAdapter(nullptr, nullptr); } - return absl::nullopt; + return std::nullopt; } absl::optional ProvideLegacyTypeInfo( absl::string_view name) const override { if (name == "test") { return test_type_info_; } - return absl::nullopt; + return std::nullopt; } private: @@ -76,8 +76,8 @@ class LegacyTypeProviderTestImpl : public LegacyTypeProvider { TEST(LegacyTypeProviderTest, EmptyTypeProviderHasProvideTypeInfo) { LegacyTypeProviderTestEmpty provider; - EXPECT_EQ(provider.ProvideLegacyType("test"), absl::nullopt); - EXPECT_EQ(provider.ProvideLegacyTypeInfo("test"), absl::nullopt); + EXPECT_EQ(provider.ProvideLegacyType("test"), std::nullopt); + EXPECT_EQ(provider.ProvideLegacyTypeInfo("test"), std::nullopt); } TEST(LegacyTypeProviderTest, NonEmptyTypeProviderProvidesSomeTypes) { @@ -85,8 +85,8 @@ TEST(LegacyTypeProviderTest, NonEmptyTypeProviderProvidesSomeTypes) { LegacyTypeProviderTestImpl provider(&test_type_info); EXPECT_TRUE(provider.ProvideLegacyType("test").has_value()); EXPECT_TRUE(provider.ProvideLegacyTypeInfo("test").has_value()); - EXPECT_EQ(provider.ProvideLegacyType("other"), absl::nullopt); - EXPECT_EQ(provider.ProvideLegacyTypeInfo("other"), absl::nullopt); + EXPECT_EQ(provider.ProvideLegacyType("other"), std::nullopt); + EXPECT_EQ(provider.ProvideLegacyTypeInfo("other"), std::nullopt); } } // namespace diff --git a/eval/public/structs/proto_message_type_adapter.cc b/eval/public/structs/proto_message_type_adapter.cc index 6a3417ba3..8c140c0c7 100644 --- a/eval/public/structs/proto_message_type_adapter.cc +++ b/eval/public/structs/proto_message_type_adapter.cc @@ -472,14 +472,14 @@ const LegacyTypeAccessApis* ProtoMessageTypeAdapter::GetAccessApis( absl::optional ProtoMessageTypeAdapter::FindFieldByName(absl::string_view field_name) const { if (descriptor_ == nullptr) { - return absl::nullopt; + return std::nullopt; } const google::protobuf::FieldDescriptor* field_descriptor = descriptor_->FindFieldByName(field_name); if (field_descriptor == nullptr) { - return absl::nullopt; + return std::nullopt; } return LegacyTypeInfoApis::FieldDescription{field_descriptor->number(), diff --git a/eval/public/structs/proto_message_type_adapter_test.cc b/eval/public/structs/proto_message_type_adapter_test.cc index 270fd3ce1..e28d76102 100644 --- a/eval/public/structs/proto_message_type_adapter_test.cc +++ b/eval/public/structs/proto_message_type_adapter_test.cc @@ -710,7 +710,7 @@ TEST(ProtoMesssageTypeAdapter, FindFieldNotFound) { "google.api.expr.runtime.TestMessage"), google::protobuf::MessageFactory::generated_factory()); - EXPECT_EQ(adapter.FindFieldByName("foo_not_a_field"), absl::nullopt); + EXPECT_EQ(adapter.FindFieldByName("foo_not_a_field"), std::nullopt); } TEST(ProtoMesssageTypeAdapter, TypeInfoMutator) { diff --git a/eval/public/structs/protobuf_descriptor_type_provider.cc b/eval/public/structs/protobuf_descriptor_type_provider.cc index 68b39c643..b5746523e 100644 --- a/eval/public/structs/protobuf_descriptor_type_provider.cc +++ b/eval/public/structs/protobuf_descriptor_type_provider.cc @@ -27,7 +27,7 @@ absl::optional ProtobufDescriptorProvider::ProvideLegacyType( absl::string_view name) const { const ProtoMessageTypeAdapter* result = GetTypeAdapter(name); if (result == nullptr) { - return absl::nullopt; + return std::nullopt; } // ProtoMessageTypeAdapter provides apis for both access and mutation. return LegacyTypeAdapter(result, result); @@ -38,7 +38,7 @@ ProtobufDescriptorProvider::ProvideLegacyTypeInfo( absl::string_view name) const { const ProtoMessageTypeAdapter* result = GetTypeAdapter(name); if (result == nullptr) { - return absl::nullopt; + return std::nullopt; } return result; } diff --git a/eval/public/structs/trivial_legacy_type_info_test.cc b/eval/public/structs/trivial_legacy_type_info_test.cc index 9b4840373..9cc6e4916 100644 --- a/eval/public/structs/trivial_legacy_type_info_test.cc +++ b/eval/public/structs/trivial_legacy_type_info_test.cc @@ -56,9 +56,9 @@ TEST(TrivialTypeInfo, FindFieldByName) { TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.FindFieldByName("foo"), absl::nullopt); + EXPECT_EQ(info.FindFieldByName("foo"), std::nullopt); EXPECT_EQ(TrivialTypeInfo::GetInstance()->FindFieldByName("foo"), - absl::nullopt); + std::nullopt); } } // namespace From b548c7de3cba492909928acec9fbc3542bb02788 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 23 Jun 2026 23:58:24 -0700 Subject: [PATCH 092/350] No public description PiperOrigin-RevId: 937135148 --- common/source.cc | 14 ++++---- common/source_test.cc | 28 ++++++++-------- common/type.cc | 8 ++--- common/type_introspector.cc | 20 ++++++------ common/type_proto.cc | 2 +- common/type_reflector_test.cc | 60 +++++++++++++++++------------------ 6 files changed, 66 insertions(+), 66 deletions(-) diff --git a/common/source.cc b/common/source.cc index 8c32ad6ba..5fa4cca0e 100644 --- a/common/source.cc +++ b/common/source.cc @@ -483,26 +483,26 @@ absl::optional Source::GetLocation( return SourceLocation{line_and_offset->first, position - line_and_offset->second}; } - return absl::nullopt; + return std::nullopt; } absl::optional Source::GetPosition( const SourceLocation& location) const { if (ABSL_PREDICT_FALSE(location.line < 1 || location.column < 0)) { - return absl::nullopt; + return std::nullopt; } if (auto position = FindLinePosition(location.line); ABSL_PREDICT_TRUE(position.has_value())) { return *position + location.column; } - return absl::nullopt; + return std::nullopt; } absl::optional Source::Snippet(int32_t line) const { auto content = this->content(); auto start = FindLinePosition(line); if (ABSL_PREDICT_FALSE(!start.has_value() || content.empty())) { - return absl::nullopt; + return std::nullopt; } auto end = FindLinePosition(line + 1); if (end.has_value()) { @@ -554,7 +554,7 @@ std::string Source::DisplayErrorLocation(SourceLocation location) const { absl::optional Source::FindLinePosition(int32_t line) const { if (ABSL_PREDICT_FALSE(line < 1)) { - return absl::nullopt; + return std::nullopt; } if (line == 1) { return SourcePosition{0}; @@ -563,13 +563,13 @@ absl::optional Source::FindLinePosition(int32_t line) const { if (ABSL_PREDICT_TRUE(line <= static_cast(line_offsets.size()))) { return line_offsets[static_cast(line - 2)]; } - return absl::nullopt; + return std::nullopt; } absl::optional> Source::FindLine( SourcePosition position) const { if (ABSL_PREDICT_FALSE(position < 0)) { - return absl::nullopt; + return std::nullopt; } int32_t line = 1; const auto line_offsets = this->line_offsets(); diff --git a/common/source_test.cc b/common/source_test.cc index 2a3b78893..30a2ce9b0 100644 --- a/common/source_test.cc +++ b/common/source_test.cc @@ -81,37 +81,37 @@ TEST(StringSource, PositionAndLocation) { Optional(Eq(SourceLocation{int32_t{1}, int32_t{2}}))); EXPECT_THAT(source->GetLocation(*end), Optional(Eq(SourceLocation{int32_t{3}, int32_t{2}}))); - EXPECT_THAT(source->GetLocation(-1), Eq(absl::nullopt)); + EXPECT_THAT(source->GetLocation(-1), Eq(std::nullopt)); EXPECT_THAT(source->content().ToString(*start, *end), Eq("d &&\n\t b.c.arg(10) &&\n\t ")); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{0}, int32_t{0}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{1}, int32_t{-1}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{4}, int32_t{0}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); } TEST(StringSource, SnippetSingle) { ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello, world", "one-line-test")); EXPECT_THAT(source->Snippet(1), Optional(Eq("hello, world"))); - EXPECT_THAT(source->Snippet(2), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(2), Eq(std::nullopt)); } TEST(StringSource, SnippetMulti) { ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello\nworld\nmy\nbub\n", "four-line-test")); - EXPECT_THAT(source->Snippet(0), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(0), Eq(std::nullopt)); EXPECT_THAT(source->Snippet(1), Optional(Eq("hello"))); EXPECT_THAT(source->Snippet(2), Optional(Eq("world"))); EXPECT_THAT(source->Snippet(3), Optional(Eq("my"))); EXPECT_THAT(source->Snippet(4), Optional(Eq("bub"))); EXPECT_THAT(source->Snippet(5), Optional(Eq(""))); - EXPECT_THAT(source->Snippet(6), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(6), Eq(std::nullopt)); } TEST(CordSource, Description) { @@ -150,17 +150,17 @@ TEST(CordSource, PositionAndLocation) { Optional(Eq(SourceLocation{int32_t{1}, int32_t{2}}))); EXPECT_THAT(source->GetLocation(*end), Optional(Eq(SourceLocation{int32_t{3}, int32_t{2}}))); - EXPECT_THAT(source->GetLocation(-1), Eq(absl::nullopt)); + EXPECT_THAT(source->GetLocation(-1), Eq(std::nullopt)); EXPECT_THAT(source->content().ToString(*start, *end), Eq("d &&\n\t b.c.arg(10) &&\n\t ")); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{0}, int32_t{0}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{1}, int32_t{-1}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(source->GetPosition(SourceLocation{int32_t{4}, int32_t{0}}), - Eq(absl::nullopt)); + Eq(std::nullopt)); } TEST(CordSource, SnippetSingle) { @@ -168,7 +168,7 @@ TEST(CordSource, SnippetSingle) { NewSource(absl::Cord("hello, world"), "one-line-test")); EXPECT_THAT(source->Snippet(1), Optional(Eq("hello, world"))); - EXPECT_THAT(source->Snippet(2), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(2), Eq(std::nullopt)); } TEST(CordSource, SnippetMulti) { @@ -176,13 +176,13 @@ TEST(CordSource, SnippetMulti) { auto source, NewSource(absl::Cord("hello\nworld\nmy\nbub\n"), "four-line-test")); - EXPECT_THAT(source->Snippet(0), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(0), Eq(std::nullopt)); EXPECT_THAT(source->Snippet(1), Optional(Eq("hello"))); EXPECT_THAT(source->Snippet(2), Optional(Eq("world"))); EXPECT_THAT(source->Snippet(3), Optional(Eq("my"))); EXPECT_THAT(source->Snippet(4), Optional(Eq("bub"))); EXPECT_THAT(source->Snippet(5), Optional(Eq(""))); - EXPECT_THAT(source->Snippet(6), Eq(absl::nullopt)); + EXPECT_THAT(source->Snippet(6), Eq(std::nullopt)); } TEST(Source, DisplayErrorLocationBasic) { diff --git a/common/type.cc b/common/type.cc index 684c5ba09..9ea85954c 100644 --- a/common/type.cc +++ b/common/type.cc @@ -158,7 +158,7 @@ absl::optional GetOrNullopt(const common_internal::TypeVariant& variant) { if (const auto* alt = absl::get_if(&variant); alt != nullptr) { return *alt; } - return absl::nullopt; + return std::nullopt; } } // namespace @@ -243,7 +243,7 @@ absl::optional Type::AsOptional() const { if (auto maybe_opaque = AsOpaque(); maybe_opaque.has_value()) { return maybe_opaque->AsOptional(); } - return absl::nullopt; + return std::nullopt; } absl::optional Type::AsString() const { @@ -263,7 +263,7 @@ absl::optional Type::AsStruct() const { if (const auto* alt = absl::get_if(&variant_); alt != nullptr) { return *alt; } - return absl::nullopt; + return std::nullopt; } absl::optional Type::AsTimestamp() const { @@ -603,7 +603,7 @@ absl::optional StructTypeField::AsMessage() const { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } StructTypeField::operator MessageTypeField() const { diff --git a/common/type_introspector.cc b/common/type_introspector.cc index 26f53685e..3846ab58b 100644 --- a/common/type_introspector.cc +++ b/common/type_introspector.cc @@ -104,7 +104,7 @@ struct WellKnownType { auto it = std::lower_bound(fields_by_name.begin(), fields_by_name.end(), name, FieldNameComparer{}); if (it == fields_by_name.end() || it->name() != name) { - return absl::nullopt; + return std::nullopt; } return *it; } @@ -114,7 +114,7 @@ struct WellKnownType { auto it = std::lower_bound(fields_by_number.begin(), fields_by_number.end(), number, FieldNumberComparer{}); if (it == fields_by_number.end() || it->number() != number) { - return absl::nullopt; + return std::nullopt; } return *it; } @@ -216,25 +216,25 @@ const WellKnownTypesMap& GetWellKnownTypesMap() { absl::StatusOr> TypeIntrospector::FindTypeImpl( absl::string_view) const { - return absl::nullopt; + return std::nullopt; } absl::StatusOr> TypeIntrospector::FindEnumConstantImpl(absl::string_view, absl::string_view) const { - return absl::nullopt; + return std::nullopt; } absl::StatusOr> TypeIntrospector::FindStructTypeFieldByNameImpl(absl::string_view, absl::string_view) const { - return absl::nullopt; + return std::nullopt; } absl::StatusOr< absl::optional>> TypeIntrospector::ListFieldsForStructTypeImpl(absl::string_view) const { - return absl::nullopt; + return std::nullopt; } absl::optional FindWellKnownType(absl::string_view name) { @@ -242,7 +242,7 @@ absl::optional FindWellKnownType(absl::string_view name) { if (auto it = well_known_types.find(name); it != well_known_types.end()) { return it->second.type; } - return absl::nullopt; + return std::nullopt; } absl::optional FindWellKnownTypeEnumConstant( @@ -251,7 +251,7 @@ absl::optional FindWellKnownTypeEnumConstant( return TypeIntrospector::EnumConstant{ IntType{}, "google.protobuf.NullValue", "NULL_VALUE", 0}; } - return absl::nullopt; + return std::nullopt; } absl::optional FindWellKnownTypeFieldByName( @@ -260,7 +260,7 @@ absl::optional FindWellKnownTypeFieldByName( if (auto it = well_known_types.find(type); it != well_known_types.end()) { return it->second.FieldByName(name); } - return absl::nullopt; + return std::nullopt; } absl::optional> @@ -268,7 +268,7 @@ ListFieldsForWellKnownType(absl::string_view type) { const auto& well_known_types = GetWellKnownTypesMap(); auto it = well_known_types.find(type); if (it == well_known_types.end()) { - return absl::nullopt; + return std::nullopt; } // The fields are not normally gettable. return {}; diff --git a/common/type_proto.cc b/common/type_proto.cc index 66c16689d..b6b66f73a 100644 --- a/common/type_proto.cc +++ b/common/type_proto.cc @@ -71,7 +71,7 @@ absl::optional MaybeWellKnownType(absl::string_view type_name) { return it->second; } - return absl::nullopt; + return std::nullopt; } absl::Status TypeToProtoInternal(const cel::Type& type, diff --git a/common/type_reflector_test.cc b/common/type_reflector_test.cc index f2ff2c322..d9c855e4b 100644 --- a/common/type_reflector_test.cc +++ b/common/type_reflector_test.cc @@ -210,7 +210,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BoolValue) { internal::GetTestingMessageFactory(), "google.protobuf.BoolValue"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -218,7 +218,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BoolValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, BoolValue(true)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -236,7 +236,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int32Value) { internal::GetTestingMessageFactory(), "google.protobuf.Int32Value"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -248,7 +248,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int32Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kOutOfRange))))); EXPECT_THAT(builder->SetFieldByNumber(1, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -270,7 +270,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int64Value) { internal::GetTestingMessageFactory(), "google.protobuf.Int64Value"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -278,7 +278,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int64Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -296,7 +296,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt32Value) { internal::GetTestingMessageFactory(), "google.protobuf.UInt32Value"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", UintValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", UintValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -308,7 +308,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt32Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kOutOfRange))))); EXPECT_THAT(builder->SetFieldByNumber(1, UintValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, UintValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -330,7 +330,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt64Value) { internal::GetTestingMessageFactory(), "google.protobuf.UInt64Value"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", UintValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", UintValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -338,7 +338,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt64Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, UintValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, UintValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -356,7 +356,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_FloatValue) { internal::GetTestingMessageFactory(), "google.protobuf.FloatValue"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", DoubleValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", DoubleValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -364,7 +364,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_FloatValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, DoubleValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, DoubleValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -382,7 +382,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_DoubleValue) { internal::GetTestingMessageFactory(), "google.protobuf.DoubleValue"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", DoubleValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", DoubleValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -390,7 +390,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_DoubleValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, DoubleValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, DoubleValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -408,7 +408,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_StringValue) { internal::GetTestingMessageFactory(), "google.protobuf.StringValue"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", StringValue("foo")), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", StringValue("foo")), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -416,7 +416,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_StringValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, StringValue("foo")), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, StringValue("foo")), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -434,7 +434,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BytesValue) { internal::GetTestingMessageFactory(), "google.protobuf.BytesValue"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("value", BytesValue("foo")), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", BytesValue("foo")), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -442,7 +442,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BytesValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, BytesValue("foo")), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, BytesValue("foo")), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -460,7 +460,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { internal::GetTestingMessageFactory(), "google.protobuf.Duration"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("seconds", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -468,7 +468,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByName("nanos", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName( "nanos", IntValue(std::numeric_limits::max())), IsOkAndHolds(Optional( @@ -477,7 +477,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(3, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -485,7 +485,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(2, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber( 2, IntValue(std::numeric_limits::max())), IsOkAndHolds(Optional( @@ -505,7 +505,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { internal::GetTestingMessageFactory(), "google.protobuf.Timestamp"); ASSERT_THAT(builder, NotNull()); EXPECT_THAT(builder->SetFieldByName("seconds", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -513,7 +513,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByName("nanos", IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName( "nanos", IntValue(std::numeric_limits::max())), IsOkAndHolds(Optional( @@ -522,7 +522,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(1, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(3, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -530,7 +530,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(2, IntValue(1)), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber( 2, IntValue(std::numeric_limits::max())), IsOkAndHolds(Optional( @@ -552,7 +552,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { EXPECT_THAT(builder->SetFieldByName( "type_url", StringValue("type.googleapis.com/google.protobuf.BoolValue")), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -560,14 +560,14 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByName("value", BytesValue()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT( builder->SetFieldByNumber( 1, StringValue("type.googleapis.com/google.protobuf.BoolValue")), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(3, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -575,7 +575,7 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); EXPECT_THAT(builder->SetFieldByNumber(2, BytesValue()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(2, BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); From 7f3e2da8e20d11e5037e3edd95541c09410bff52 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 24 Jun 2026 00:57:41 -0700 Subject: [PATCH 093/350] No public description PiperOrigin-RevId: 937159464 --- extensions/protobuf/internal/qualify.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/protobuf/internal/qualify.cc b/extensions/protobuf/internal/qualify.cc index dba4f44ae..37ad30011 100644 --- a/extensions/protobuf/internal/qualify.cc +++ b/extensions/protobuf/internal/qualify.cc @@ -145,7 +145,7 @@ absl::StatusOr> LookupMapValu bool found = cel::extensions::protobuf_internal::LookupMapValue( *reflection, *message, *field_desc, proto_key, &value_ref); if (!found) { - return absl::nullopt; + return std::nullopt; } return value_ref; } From 43960d06c38367520733fc2054688ec5fe447b28 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 24 Jun 2026 07:01:10 -0700 Subject: [PATCH 094/350] No public description PiperOrigin-RevId: 937312837 --- common/types/opaque_type.cc | 2 +- common/types/struct_type.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/types/opaque_type.cc b/common/types/opaque_type.cc index 002319d1d..9c58e8289 100644 --- a/common/types/opaque_type.cc +++ b/common/types/opaque_type.cc @@ -98,7 +98,7 @@ absl::optional OpaqueType::AsOptional() const { if (IsOptional()) { return OptionalType(absl::in_place, *this); } - return absl::nullopt; + return std::nullopt; } OptionalType OpaqueType::GetOptional() const { diff --git a/common/types/struct_type.cc b/common/types/struct_type.cc index a1be1f786..69f531a2f 100644 --- a/common/types/struct_type.cc +++ b/common/types/struct_type.cc @@ -61,7 +61,7 @@ absl::optional StructType::AsMessage() const { if (const auto* alt = absl::get_if(&variant_); alt != nullptr) { return *alt; } - return absl::nullopt; + return std::nullopt; } MessageType StructType::GetMessage() const { From 7d327d6e55f31327087240adcc26d890729ee8d5 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 29 Jun 2026 11:36:15 -0700 Subject: [PATCH 095/350] Add a cel::Source overload for cel::Compiler. PiperOrigin-RevId: 939942572 --- compiler/BUILD | 1 + compiler/compiler.h | 33 +++++++++++++++++++++++++++++-- compiler/compiler_factory.cc | 27 +++++++++++-------------- compiler/compiler_factory_test.cc | 14 +++++++++++++ 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/compiler/BUILD b/compiler/BUILD index d4a0ab4ac..2e3f7a2ba 100644 --- a/compiler/BUILD +++ b/compiler/BUILD @@ -25,6 +25,7 @@ cc_library( "//checker:type_checker", "//checker:type_checker_builder", "//checker:validation_result", + "//common:source", "//parser:options", "//parser:parser_interface", "//validator", diff --git a/compiler/compiler.h b/compiler/compiler.h index 27237df60..174912f71 100644 --- a/compiler/compiler.h +++ b/compiler/compiler.h @@ -27,6 +27,7 @@ #include "checker/type_checker.h" #include "checker/type_checker_builder.h" #include "checker/validation_result.h" +#include "common/source.h" #include "parser/options.h" #include "parser/parser_interface.h" #include "validator/validator.h" @@ -129,9 +130,18 @@ class Compiler { public: virtual ~Compiler() = default; - virtual absl::StatusOr Compile( + absl::StatusOr Compile( + const Source& source, google::protobuf::Arena* absl_nullable arena) const { + return CompileImpl(source, arena); + } + + absl::StatusOr Compile(const Source& source) const { + return CompileImpl(source, nullptr); + } + + absl::StatusOr Compile( absl::string_view source, absl::string_view description, - google::protobuf::Arena* absl_nullable arena) const = 0; + google::protobuf::Arena* absl_nullable arena) const; absl::StatusOr Compile(absl::string_view source) const { return Compile(source, "", nullptr); @@ -159,8 +169,27 @@ class Compiler { // The returned builder does not share state with the compiler and may be // modified independently. virtual std::unique_ptr ToBuilder() const = 0; + + protected: + virtual absl::StatusOr CompileImpl( + const Source& source, google::protobuf::Arena* absl_nullable arena) const = 0; }; +inline absl::StatusOr Compiler::Compile( + absl::string_view source, absl::string_view description, + google::protobuf::Arena* absl_nullable arena) const { + absl::StatusOr source_obj = + NewSource(source, std::string(description)); + if (!source_obj.ok()) { + return source_obj.status(); + } + absl::StatusOr result = CompileImpl(**source_obj, arena); + if (result.ok()) { + result->SetSource(std::move(*source_obj)); + } + return result; +} + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMPILER_COMPILER_INTERFACE_H_ diff --git a/compiler/compiler_factory.cc b/compiler/compiler_factory.cc index ed22c5630..a8a44461c 100644 --- a/compiler/compiler_factory.cc +++ b/compiler/compiler_factory.cc @@ -54,14 +54,18 @@ class CompilerImpl : public Compiler { validator_(std::move(validator)), options_(options) {} - absl::StatusOr Compile( - absl::string_view expression, absl::string_view description, - google::protobuf::Arena* arena) const override { - CEL_ASSIGN_OR_RETURN(auto source, - cel::NewSource(expression, std::string(description))); + std::unique_ptr ToBuilder() const override; + + const TypeChecker& GetTypeChecker() const override { return *type_checker_; } + const Parser& GetParser() const override { return *parser_; } + const Validator& GetValidator() const override { return validator_; } + + protected: + absl::StatusOr CompileImpl( + const Source& source, google::protobuf::Arena* arena) const override { std::vector parse_issues; absl::StatusOr> ast = - parser_->Parse(*source, &parse_issues); + parser_->Parse(source, &parse_issues); if (!ast.ok()) { if (!options_.adapt_parser_errors || ast.status().code() != absl::StatusCode::kInvalidArgument || @@ -74,26 +78,17 @@ class CompilerImpl : public Compiler { check_issues.push_back(TypeCheckIssue::CreateError( issue.location(), std::string(issue.message()))); } - ValidationResult result(std::move(check_issues)); - result.SetSource(std::move(source)); - return result; + return ValidationResult(std::move(check_issues)); } CEL_ASSIGN_OR_RETURN(ValidationResult result, type_checker_->Check(*std::move(ast), arena)); - result.SetSource(std::move(source)); if (!validator_.validations().empty()) { validator_.UpdateValidationResult(result); } return result; } - std::unique_ptr ToBuilder() const override; - - const TypeChecker& GetTypeChecker() const override { return *type_checker_; } - const Parser& GetParser() const override { return *parser_; } - const Validator& GetValidator() const override { return validator_; } - private: std::unique_ptr type_checker_; std::unique_ptr parser_; diff --git a/compiler/compiler_factory_test.cc b/compiler/compiler_factory_test.cc index 035fd8aa6..3f83f5bf2 100644 --- a/compiler/compiler_factory_test.cc +++ b/compiler/compiler_factory_test.cc @@ -427,5 +427,19 @@ TEST(CompilerFactoryTest, ReturnsIssuesFromParser) { EXPECT_THAT(result.GetIssues(), testing::Not(testing::IsEmpty())); } +TEST(CompilerFactoryTest, CompileSourceOverload) { + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool())); + + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("1 + 2")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, compiler->Compile(*source)); + + EXPECT_TRUE(result.IsValid()); +} + } // namespace } // namespace cel From eb8a8943f61d334cc523750ee66886a2b249744b Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 29 Jun 2026 12:25:41 -0700 Subject: [PATCH 096/350] Add support for creating a subspan view of a cel::Source. PiperOrigin-RevId: 939969141 --- common/source.cc | 45 +++++++++++++++++++++++++++++++++++++++++++ common/source.h | 32 ++++++++++++++++++++++++++++++ common/source_test.cc | 39 +++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/common/source.cc b/common/source.cc index 5fa4cca0e..f268bc833 100644 --- a/common/source.cc +++ b/common/source.cc @@ -585,6 +585,51 @@ absl::optional> Source::FindLine( return std::make_pair(line, line_offsets[static_cast(line) - 2]); } +SourceSubrange::SourceSubrange(const Source& source, SourceRange range) + : source_(source), range_(range) { + SourcePosition size = source_.content().size(); + ABSL_DCHECK(range_.begin >= 0); + ABSL_DCHECK(range_.begin <= size); + ABSL_DCHECK(range_.end >= range_.begin); + ABSL_DCHECK(range_.end <= size); + if (range_.begin < 0) { + range_.begin = 0; + } + if (range_.begin > size) { + range_.begin = size; + } + if (range_.end < range_.begin) { + range_.end = range_.begin; + } + if (range_.end > size) { + range_.end = size; + } + for (const auto& line_offset : source_.line_offsets()) { + if (line_offset > range_.begin && line_offset <= range_.end) { + line_offsets_.push_back(line_offset - range_.begin); + } + } + line_offsets_.push_back(range_.end - range_.begin + 1); +} + +SourceContentView SourceSubrange::content() const { + auto parent_content = source_.content(); + if (parent_content.empty() || range_.begin >= range_.end) { + return EmptyContentView(); + } + return absl::visit( + [this](auto view) { + return SourceContentView( + view.subspan(static_cast(range_.begin), + static_cast(range_.end - range_.begin))); + }, + parent_content.view_); +} + +absl::Span SourceSubrange::line_offsets() const { + return absl::MakeConstSpan(line_offsets_); +} + absl::StatusOr NewSource(absl::string_view content, std::string description) { return common_internal::NewSourceImpl(std::move(description), content, diff --git a/common/source.h b/common/source.h index 6453363a8..b3968453f 100644 --- a/common/source.h +++ b/common/source.h @@ -22,6 +22,7 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" +#include "absl/container/inlined_vector.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" @@ -36,6 +37,7 @@ class SourceImpl; } // namespace common_internal class Source; +class SourceSubrange; // SourcePosition represents an offset in source text. using SourcePosition = int32_t; @@ -94,6 +96,7 @@ class SourceContentView final { private: friend class Source; + friend class SourceSubrange; constexpr SourceContentView() = default; @@ -178,6 +181,7 @@ class Source { private: friend class common_internal::SourceImpl; + friend class SourceSubrange; Source() = default; @@ -187,6 +191,34 @@ class Source { SourcePosition position) const; }; +// `SourceSubrange` is a view of a subrange fo an underlying `Source` object. +// Intended to be used when the CEL expression is embedded in a larger text +// representation (such as a.celpolicy). +// +// The parent `Source` must outlive this object. +class SourceSubrange final : public Source { + public: + SourceSubrange(const Source& source ABSL_ATTRIBUTE_LIFETIME_BOUND, + SourceRange range); + + absl::string_view description() const ABSL_ATTRIBUTE_LIFETIME_BOUND override { + return source_.description(); + } + + // Returns a view of the underlying expression text. + ContentView content() const ABSL_ATTRIBUTE_LIFETIME_BOUND override; + + // Returns a `absl::Span` of `SourcePosition` which represent the positions + // where new lines occur. + absl::Span line_offsets() const + ABSL_ATTRIBUTE_LIFETIME_BOUND override; + + private: + const Source& source_; + SourceRange range_; + absl::InlinedVector line_offsets_; +}; + using SourcePtr = std::unique_ptr; absl::StatusOr NewSource( diff --git a/common/source_test.cc b/common/source_test.cc index 30a2ce9b0..e620d3dd8 100644 --- a/common/source_test.cc +++ b/common/source_test.cc @@ -223,5 +223,44 @@ TEST(Source, DisplayErrorLocationFullWidth) { "\n | ..^"); } +TEST(SourceSubrange, Description) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello world", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{0, 5}); + EXPECT_THAT(subrange.description(), Eq("subrange-test")); +} + +TEST(SourceSubrange, Content) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello world", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{6, 11}); + EXPECT_THAT(subrange.content().ToString(), Eq("world")); +} + +TEST(SourceSubrange, ContentEmpty) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("hello world", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{5, 5}); + EXPECT_THAT(subrange.content().ToString(), Eq("")); +} + +TEST(SourceSubrange, LineOffsetsNoNewlines) { + ASSERT_OK_AND_ASSIGN(auto source, + NewSource("hello\nworld\n", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{0, 5}); + EXPECT_THAT(subrange.line_offsets(), ElementsAre(6)); +} + +TEST(SourceSubrange, LineOffsetsWithNewlines) { + ASSERT_OK_AND_ASSIGN(auto source, + NewSource("hello\nworld\ncel", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{0, 11}); + EXPECT_THAT(subrange.line_offsets(), ElementsAre(6, 12)); +} + +TEST(SourceSubrange, LineOffsetsMiddleSubrange) { + ASSERT_OK_AND_ASSIGN(auto source, + NewSource("hello\nworld\ncel\ncpp", "subrange-test")); + SourceSubrange subrange(*source, SourceRange{6, 15}); + EXPECT_THAT(subrange.line_offsets(), ElementsAre(6, 10)); +} + } // namespace } // namespace cel From fff5f34e279a12c1140d5c2744881dc36ff05432 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 30 Jun 2026 00:06:14 -0700 Subject: [PATCH 097/350] No public description PiperOrigin-RevId: 940271432 --- common/signature.cc | 2 +- internal/time.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/signature.cc b/common/signature.cc index e497e780d..54d312777 100644 --- a/common/signature.cc +++ b/common/signature.cc @@ -327,7 +327,7 @@ absl::optional ParseBuiltinOrWrapper(std::string_view name_str) { std::make_unique(DynTypeSpec()))); } - return absl::nullopt; + return std::nullopt; } std::string Unescape(std::string_view str) { diff --git a/internal/time.cc b/internal/time.cc index 45945613d..c42212b68 100644 --- a/internal/time.cc +++ b/internal/time.cc @@ -42,7 +42,7 @@ absl::Duration MaxDuration() { // intent is to widen the CEL spec to support the larger range and match // google.protobuf.Duration from protocol buffer messages, which this // implementation currently supports. - // TODO(google/cel-spec/issues/214): revisit + // TODO(cel-expr/cel-spec/issues/214): revisit return absl::Seconds(google::protobuf::util::TimeUtil::kDurationMaxSeconds) + absl::Nanoseconds(google::protobuf::util::TimeUtil::kDurationMaxNanoseconds); } @@ -52,7 +52,7 @@ absl::Duration MinDuration() { // intent is to widen the CEL spec to support the larger range and match // google.protobuf.Duration from protocol buffer messages, which this // implementation currently supports. - // TODO(google/cel-spec/issues/214): revisit + // TODO(cel-expr/cel-spec/issues/214): revisit return absl::Seconds(google::protobuf::util::TimeUtil::kDurationMinSeconds) + absl::Nanoseconds(google::protobuf::util::TimeUtil::kDurationMinNanoseconds); } From a925812307420a52d6a49fc299b6d6f96e405f7a Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 30 Jun 2026 11:46:08 -0700 Subject: [PATCH 098/350] Add support for estimating original source positions. Ported from the go implementation. PiperOrigin-RevId: 940574177 --- common/source.cc | 1 + policy/BUILD | 3 + policy/cel_policy.cc | 48 +++++- policy/cel_policy.h | 15 +- policy/cel_policy_parse_result.cc | 4 + policy/compiler.cc | 85 ++++++++-- policy/compiler_test.cc | 159 ++++++++++++++++++ policy/internal/BUILD | 22 +++ .../internal/yaml_string_element_scanner.cc | 127 ++++++++++++++ policy/internal/yaml_string_element_scanner.h | 40 +++++ .../yaml_string_element_scanner_test.cc | 105 ++++++++++++ policy/yaml_policy_parser.cc | 12 ++ 12 files changed, 597 insertions(+), 24 deletions(-) create mode 100644 policy/internal/yaml_string_element_scanner.cc create mode 100644 policy/internal/yaml_string_element_scanner.h create mode 100644 policy/internal/yaml_string_element_scanner_test.cc diff --git a/common/source.cc b/common/source.cc index f268bc833..81b1ada9a 100644 --- a/common/source.cc +++ b/common/source.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/policy/BUILD b/policy/BUILD index 19195be2b..cace76f5b 100644 --- a/policy/BUILD +++ b/policy/BUILD @@ -84,6 +84,7 @@ cc_library( ":cel_policy_parser", "//common:source", "//internal:status_macros", + "//policy/internal:yaml_string_element_scanner", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -190,6 +191,7 @@ cc_test( ":compiler", ":yaml_policy_parser", "//common:ast", + "//common:ast_proto", "//common:decl", "//common:navigable_ast", "//common:source", @@ -211,6 +213,7 @@ cc_test( "//runtime:runtime_builder", "//runtime:runtime_options", "//runtime:standard_runtime_builder_factory", + "//tools:cel_unparser", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", diff --git a/policy/cel_policy.cc b/policy/cel_policy.cc index c2d97edeb..0f5edcc19 100644 --- a/policy/cel_policy.cc +++ b/policy/cel_policy.cc @@ -59,25 +59,53 @@ std::string IndentBlock(absl::string_view text) { void CelPolicySource::NoteSourcePosition(CelPolicyElementId id, SourcePosition position) { - source_positions_[id] = position; + source_info_[id].position = position; +} + +void CelPolicySource::NoteSourceRange(CelPolicyElementId id, + std::optional range, + bool quoted) { + ElementSourceInfo& info = source_info_[id]; + info.range = range; + info.quoted = quoted; + if (range.has_value() && info.position == -1) { + info.position = range->begin; + } } std::optional CelPolicySource::GetSourcePosition( CelPolicyElementId id) const { - auto it = source_positions_.find(id); - if (it == source_positions_.end()) { + auto it = source_info_.find(id); + if (it == source_info_.end() || it->second.position == -1) { return std::nullopt; } - return it->second; + return it->second.position; +} + +std::optional CelPolicySource::GetSourceRange( + CelPolicyElementId id) const { + auto it = source_info_.find(id); + if (it == source_info_.end() || !it->second.range.has_value()) { + return std::nullopt; + } + return it->second.range; +} + +std::optional CelPolicySource::IsQuoted(CelPolicyElementId id) const { + auto it = source_info_.find(id); + if (it == source_info_.end() || !it->second.range.has_value()) { + return std::nullopt; + } + return it->second.quoted; } std::optional CelPolicySource::GetSourceLocation( CelPolicyElementId id) const { - auto it = source_positions_.find(id); - if (it == source_positions_.end()) { + auto it = source_info_.find(id); + if (it == source_info_.end() || it->second.position == -1) { return std::nullopt; } - return policy_source_->GetLocation(it->second); + return policy_source_->GetLocation(it->second.position); } std::string CelPolicySource::DebugString() const { @@ -85,8 +113,10 @@ std::string CelPolicySource::DebugString() const { // Sort the source elements in descending order of position std::vector> sorted_positions; - for (const auto& pair : source_positions_) { - sorted_positions.push_back(pair); + for (const auto& [id, info] : source_info_) { + if (info.position != -1) { + sorted_positions.push_back({id, info.position}); + } } std::sort(sorted_positions.begin(), sorted_positions.end(), [](const auto& a, const auto& b) { diff --git a/policy/cel_policy.h b/policy/cel_policy.h index af8f7c977..2e8ab1414 100644 --- a/policy/cel_policy.h +++ b/policy/cel_policy.h @@ -34,6 +34,12 @@ namespace cel { using CelPolicyElementId = int32_t; +struct ElementSourceInfo { + SourcePosition position = -1; + std::optional range; + bool quoted = false; +}; + class CelPolicySource { public: explicit CelPolicySource(cel::SourcePtr policy_source) @@ -43,15 +49,22 @@ class CelPolicySource { void NoteSourcePosition(CelPolicyElementId id, SourcePosition position); + void NoteSourceRange(CelPolicyElementId id, std::optional range, + bool quoted); + std::optional GetSourcePosition(CelPolicyElementId id) const; + std::optional GetSourceRange(CelPolicyElementId id) const; + + std::optional IsQuoted(CelPolicyElementId id) const; + std::optional GetSourceLocation(CelPolicyElementId id) const; std::string DebugString() const; private: cel::SourcePtr policy_source_; - absl::flat_hash_map source_positions_; + absl::flat_hash_map source_info_; }; class ValueString { diff --git a/policy/cel_policy_parse_result.cc b/policy/cel_policy_parse_result.cc index 32d6431bb..d8c9bb2b0 100644 --- a/policy/cel_policy_parse_result.cc +++ b/policy/cel_policy_parse_result.cc @@ -52,8 +52,12 @@ std::string CelPolicyIssue::ToDisplayString( std::string snippet; if (source != nullptr) { if (relative_position_) { + std::optional range = source->GetSourceRange(element_id_); std::optional base = source->GetSourcePosition(element_id_); + if (range.has_value()) { + base = range->begin; + } if (element_id_ == -1) { base.emplace(0); } diff --git a/policy/compiler.cc b/policy/compiler.cc index 7a892447c..98103dbe1 100644 --- a/policy/compiler.cc +++ b/policy/compiler.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -157,11 +158,19 @@ class IntermediateCompiledPolicy { void set_semantics(RuleSemantics semantics) { semantics_ = semantics; } RuleSemantics semantics() const { return semantics_; } + void set_policy_source(const CelPolicySource* absl_nullable src) { + policy_source_ = src; + } + const CelPolicySource* absl_nullable policy_source() const { + return policy_source_; + } + private: std::string name_; std::string display_name_; std::string description_; RuleSemantics semantics_ = RuleSemantics::kFirstMatch; + const CelPolicySource* absl_nullable policy_source_ = nullptr; CompiledRule root_rule_; }; @@ -315,6 +324,33 @@ class PolicyCompiler { return src_->content()->description(); } + absl::StatusOr CompileExpression(CelPolicyElementId id, + absl::string_view val, + const Compiler* env) { + std::unique_ptr source; + if (src_ != nullptr && src_->content() != nullptr) { + std::optional range; + range = src_->GetSourceRange(id); + bool use_subrange = !(src_->IsQuoted(id).value_or(true)); + if (range.has_value() && use_subrange) { + source = std::make_unique(*src_->content(), *range); + } + } + + if (source == nullptr) { + // For quoted strings, the source should be generated from the interpreted + // YAML value. + CEL_ASSIGN_OR_RETURN( + source, cel::NewSource(val, std::string(GetSourceDescription()))); + } + auto result = env->Compile(*source, &arena_); + if (!result.ok()) { + return result; + } + result->SetSource(std::move(source)); + return result; + } + void AdaptTypeCheckIssues(CelPolicyElementId id, const ValidationResult& r) { const Source* source = r.GetSource(); @@ -336,8 +372,8 @@ class PolicyCompiler { const cel::OutputBlock& output_block, const Compiler* env) { CompiledOutputBlock output; CEL_ASSIGN_OR_RETURN(auto output_validation, - env->Compile(output_block.output().value(), - GetSourceDescription(), &arena_)); + CompileExpression(output_block.output().id(), + output_block.output().value(), env)); AdaptTypeCheckIssues(output_block.output().id(), output_validation); cel::Type result_type = DynType(); @@ -352,9 +388,10 @@ class PolicyCompiler { } } if (output_block.explanation().has_value()) { - CEL_ASSIGN_OR_RETURN(auto explanation_validation, - env->Compile(output_block.explanation()->value(), - GetSourceDescription(), &arena_)); + CEL_ASSIGN_OR_RETURN( + auto explanation_validation, + CompileExpression(output_block.explanation()->id(), + output_block.explanation()->value(), env)); AdaptTypeCheckIssues(output_block.explanation()->id(), explanation_validation); if (explanation_validation.IsValid()) { @@ -378,8 +415,8 @@ class PolicyCompiler { c_match.id = match.id(); if (match.condition().has_value()) { CEL_ASSIGN_OR_RETURN(auto validation, - env->Compile(match.condition()->value(), - GetSourceDescription(), &arena_)); + CompileExpression(match.condition()->id(), + match.condition()->value(), env)); AdaptTypeCheckIssues(match.condition()->id(), validation); if (validation.IsValid()) { CEL_ASSIGN_OR_RETURN(auto ast, validation.ReleaseAst()); @@ -422,9 +459,10 @@ class PolicyCompiler { continue; } std::string ident = absl::StrCat("variables.", name); - CEL_ASSIGN_OR_RETURN(auto validation, - env->Compile(variable.expression().value(), - GetSourceDescription(), &arena_)); + CEL_ASSIGN_OR_RETURN( + auto validation, + CompileExpression(variable.expression().id(), + variable.expression().value(), env)); AdaptTypeCheckIssues(variable.expression().id(), validation); if (!validation.IsValid()) { continue; @@ -480,6 +518,7 @@ class PolicyCompiler { absl::Status CompilePolicy(const CelPolicy& policy, IntermediateCompiledPolicy* out) { src_ = policy.source(); + out->set_policy_source(src_); out->set_semantics(RuleSemantics::kFirstMatch); out->set_name(policy.name().value()); out->set_display_name( @@ -513,6 +552,21 @@ class FirstMatchComposer { std::unique_ptr ReleaseAst() { return std::move(ast_); } private: + SourcePosition GetAstOffset(CelPolicyElementId id) const { + if (icp_.policy_source() == nullptr) { + return 0; + } + if (auto range = icp_.policy_source()->GetSourceRange(id); + range.has_value()) { + return range->begin; + } + if (auto pos = icp_.policy_source()->GetSourcePosition(id); + pos.has_value()) { + return *pos; + } + return 0; + } + using VariableScope = absl::flat_hash_map; std::optional ResolvePolicyVariable(absl::string_view reference); @@ -733,7 +787,8 @@ absl::StatusOr FirstMatchComposer::ComposeRule(const CompiledRule& rule, MapVariables(condition); factory_.StartCopyContext(); auto copy = factory_.Copy(condition.root_expr()); - auto source_info = factory_.RemapSourceInfo(condition.source_info()); + auto source_info = factory_.RemapSourceInfo( + condition.source_info(), GetAstOffset(match.condition->id)); factory_.MergeSourceInfo(source_info); *insertion_point = factory_.NewCall("_?_:_", std::move(copy)); insertion_point->mutable_call_expr().mutable_args().push_back( @@ -792,7 +847,8 @@ absl::StatusOr FirstMatchComposer::ComposeProduction( MapVariables(ast); factory_.StartCopyContext(); Expr to_insert = factory_.Copy(ast.root_expr()); - auto source_info = factory_.RemapSourceInfo(ast.source_info()); + auto source_info = + factory_.RemapSourceInfo(ast.source_info(), GetAstOffset(output_ast.id)); factory_.MergeSourceInfo(source_info); insertion_expr = std::move(to_insert); @@ -832,8 +888,9 @@ void FirstMatchComposer::ComposeRuleVariables(const CompiledRule& rule, MapVariables(ast); factory_.StartCopyContext(); auto insertion = factory_.Copy(ast.root_expr()); - // TODO(b/506179116): apply the position offsets here. - auto info = factory_.RemapSourceInfo(ast.source_info()); + auto info = factory_.RemapSourceInfo(ast.source_info(), + GetAstOffset(variable.ast.id)); + factory_.MergeSourceInfo(info); ABSL_DCHECK(init.has_list_expr()); int index = init.mutable_list_expr().elements().size(); init.mutable_list_expr().mutable_elements().push_back( diff --git a/policy/compiler_test.cc b/policy/compiler_test.cc index 8db494b45..6702b5002 100644 --- a/policy/compiler_test.cc +++ b/policy/compiler_test.cc @@ -24,6 +24,7 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "common/ast.h" +#include "common/ast_proto.h" #include "common/decl.h" #include "common/navigable_ast.h" #include "common/source.h" @@ -50,6 +51,7 @@ #include "runtime/runtime_builder.h" #include "runtime/runtime_options.h" #include "runtime/standard_runtime_builder_factory.h" +#include "tools/cel_unparser.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -189,6 +191,163 @@ name: cel_policy testing::HasSubstr("undeclared reference")); } +TEST(CompilerTest, DisplayErrorFormattingForInvalidExpression) { + absl::string_view yaml = R"yaml(name: cel_policy +rule: + match: + - condition: x > 0 + output: undeclared_var +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.FormatIssues(), + testing::HasSubstr( + R"err(ERROR: test.yaml:5:15: undeclared reference to 'undeclared_var' (in container '') + | output: undeclared_var + | ..............^)err")); +} + +TEST(CompilerTest, DisplayErrorFormattingForMultilineBlockLiteral) { + absl::string_view yaml = R"yaml(name: cel_policy +rule: + match: + - condition: x > 0 + output: | + undeclared_var +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.FormatIssues(), + testing::HasSubstr( + R"err(ERROR: test.yaml:6:9: undeclared reference to 'undeclared_var' (in container '') + | undeclared_var + | ........^)err")); +} + +TEST(CompilerTest, DisplayErrorFormattingForDoubleQuotedExpression) { + absl::string_view yaml = R"yaml(name: cel_policy +rule: + match: + - condition: x > 0 + output: "undeclared_var" +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + EXPECT_FALSE(result.IsValid()); + EXPECT_THAT( + result.FormatIssues(), + testing::HasSubstr( + R"err(ERROR: test.yaml:5:16: undeclared reference to 'undeclared_var' (in container '') + | output: "undeclared_var" + | ...............^)err")); +} + +TEST(CompilerTest, ComposedAstPositionsAreRelativeToMainYaml) { + absl::string_view yaml = R"yaml(name: cel_policy +rule: + variables: + - name: v1 + expression: 10 + match: + - condition: variables.v1 > 5 + output: "variables.v1 + 20" +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()); + const cel::Ast* ast = result.GetAst(); + ASSERT_NE(ast, nullptr); + const cel::Source* source = result.GetSource()->content(); + ASSERT_NE(source, nullptr); + + auto nav_ast = cel::NavigableAst::Build(ast->root_expr()); + const cel::NavigableAstNode* var_node = nullptr; + const cel::NavigableAstNode* cond_node = nullptr; + const cel::NavigableAstNode* out_node = nullptr; + + for (const cel::NavigableAstNode& node : + nav_ast.Root().DescendantsPostorder()) { + if (node.expr()->has_const_expr() && + node.expr()->const_expr().has_int_value() && + node.expr()->const_expr().int_value() == 10) { + var_node = &node; + } else if (node.expr()->has_call_expr() && + node.expr()->call_expr().function() == "_>_") { + cond_node = &node; + } else if (node.expr()->has_call_expr() && + node.expr()->call_expr().function() == "_+_") { + out_node = &node; + } + } + + ASSERT_NE(var_node, nullptr); + ASSERT_NE(cond_node, nullptr); + ASSERT_NE(out_node, nullptr); + + auto var_pos = ast->source_info().positions().find(var_node->expr()->id()); + ASSERT_NE(var_pos, ast->source_info().positions().end()); + auto var_loc = source->GetLocation(var_pos->second); + ASSERT_TRUE(var_loc.has_value()); + EXPECT_THAT( + source->DisplayErrorLocation(*var_loc), + testing::HasSubstr(" expression: 10\n | ..................^")); + + auto cond_pos = ast->source_info().positions().find(cond_node->expr()->id()); + ASSERT_NE(cond_pos, ast->source_info().positions().end()); + auto cond_loc = source->GetLocation(cond_pos->second); + ASSERT_TRUE(cond_loc.has_value()); + EXPECT_THAT(source->DisplayErrorLocation(*cond_loc), + testing::HasSubstr(" - condition: variables.v1 > 5\n | " + "..............................^")); + + auto out_pos = ast->source_info().positions().find(out_node->expr()->id()); + ASSERT_NE(out_pos, ast->source_info().positions().end()); + auto out_loc = source->GetLocation(out_pos->second); + ASSERT_TRUE(out_loc.has_value()); + EXPECT_THAT(source->DisplayErrorLocation(*out_loc), + testing::HasSubstr(" output: \"variables.v1 + 20\"\n | " + "............................^")); +} + +TEST(CompilerTest, UnparseComposedAstWithMacros) { + absl::string_view yaml = R"yaml(name: macro_policy +rule: + variables: + - name: var_inner + expression: "[1, 2].all(i, i > 0)" + - name: var_outer + expression: "[3, 4].exists(j, j > 0 && variables.var_inner)" + match: + - condition: "[5, 6].all(k, k > 0)" + output: "[1].map(m, m > 0 && variables.var_outer)" +)yaml"; + ASSERT_OK_AND_ASSIGN(auto policy, ParsePolicyFromYaml(yaml)); + ASSERT_OK_AND_ASSIGN(auto compiler, BuildTestCompiler()); + ASSERT_OK_AND_ASSIGN(auto result, CompilePolicy(*compiler, *policy)); + ASSERT_TRUE(result.IsValid()) << result.FormatIssues(); + const cel::Ast* ast = result.GetAst(); + ASSERT_NE(ast, nullptr); + + cel::expr::ParsedExpr parsed_expr; + ASSERT_THAT(cel::AstToParsedExpr(*ast, &parsed_expr), IsOk()); + ASSERT_OK_AND_ASSIGN(std::string unparsed, + google::api::expr::Unparse(parsed_expr)); + + EXPECT_EQ( + unparsed, + "cel.@block([[1, 2].all(i, i > 0), [3, 4].exists(j, j > 0 && @index0)], " + "[5, 6].all(k, k > 0) ? optional.of([1].map(m, m > 0 && @index1)) : " + "optional.none())"); +} + TEST(CompilerTest, UnreachableMatchAfterTriviallyTrueCondition) { absl::string_view yaml = R"yaml( name: cel_policy diff --git a/policy/internal/BUILD b/policy/internal/BUILD index 30f43d431..98aeaeebb 100644 --- a/policy/internal/BUILD +++ b/policy/internal/BUILD @@ -66,3 +66,25 @@ cc_test( "@com_google_absl//absl/types:span", ], ) + +cc_library( + name = "yaml_string_element_scanner", + srcs = ["yaml_string_element_scanner.cc"], + hdrs = ["yaml_string_element_scanner.h"], + deps = [ + "//common:source", + "//internal:utf8", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_test( + name = "yaml_string_element_scanner_test", + srcs = ["yaml_string_element_scanner_test.cc"], + deps = [ + ":yaml_string_element_scanner", + "//common:source", + "//internal:testing", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/policy/internal/yaml_string_element_scanner.cc b/policy/internal/yaml_string_element_scanner.cc new file mode 100644 index 000000000..06bd8e899 --- /dev/null +++ b/policy/internal/yaml_string_element_scanner.cc @@ -0,0 +1,127 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/yaml_string_element_scanner.h" + +#include + +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "internal/utf8.h" + +namespace cel::policy_internal { +namespace { + +SourceRange ScanDoubleQuotedExpression(const SourceContentView& view, + SourcePosition pos) { + SourcePosition start = pos + 1; + SourcePosition cur = start; + while (cur < view.size()) { + char32_t ch = view.at(cur); + if (ch == '\\') { + cur += 2; + continue; + } + if (ch == '"') { + break; + } + cur++; + } + return SourceRange{start, cur}; +} + +SourceRange ScanSingleQuotedExpression(const SourceContentView& view, + SourcePosition pos) { + SourcePosition start = pos + 1; + SourcePosition cur = start; + while (cur < view.size()) { + char32_t ch = view.at(cur); + if (ch == '\'') { + if (cur + 1 < view.size() && view.at(cur + 1) == '\'') { + cur += 2; + continue; + } + break; + } + cur++; + } + return SourceRange{start, cur}; +} + +SourceRange ScanPlainOrBlockExpression(const SourceContentView& view, + SourcePosition pos, + absl::string_view val) { + if (val.empty()) { + return SourceRange{pos, pos}; + } + + char32_t first_char = view.at(pos); + SourcePosition start = pos; + if (first_char == '|' || first_char == '>') { + // Skip block header line + while (start < view.size() && view.at(start) != '\n') { + start++; + } + if (start < view.size() && view.at(start) == '\n') { + start++; + } + while (start < view.size() && + (view.at(start) == ' ' || view.at(start) == '\t')) { + start++; + } + } + + SourcePosition cur = start; + absl::string_view remaining = val; + while (!remaining.empty()) { + auto [code_point, code_units] = cel::internal::Utf8Decode(remaining); + remaining.remove_prefix(code_units); + if (code_point == ' ' || code_point == '\t' || code_point == '\n') continue; + while (cur < view.size() && view.at(cur) != code_point) { + cur++; + } + if (cur < view.size()) cur++; + } + return SourceRange{start, cur}; +} + +} // namespace + +// Scans the YAML string element starting at `pos` in `view` with the parsed +// string value `val`. +// +// Returns a `YamlStringElement` reporting the starting position, optional +// source range, and whether it was quoted. +// +// TODO(b/525495513): Implement support for correctly aligning the source YAML +// to the decoded string value (the component CEL expression). +YamlStringElement ScanYamlStringElement(const SourceContentView& view, + SourcePosition pos, + absl::string_view val) { + if (pos < 0 || pos >= view.size()) { + return YamlStringElement{pos, std::nullopt, false}; + } + + char32_t first_char = view.at(pos); + if (first_char == '"') { + return YamlStringElement{pos, ScanDoubleQuotedExpression(view, pos), true}; + } + if (first_char == '\'') { + return YamlStringElement{pos, ScanSingleQuotedExpression(view, pos), true}; + } + return YamlStringElement{pos, ScanPlainOrBlockExpression(view, pos, val), + false}; +} + +} // namespace cel::policy_internal diff --git a/policy/internal/yaml_string_element_scanner.h b/policy/internal/yaml_string_element_scanner.h new file mode 100644 index 000000000..f4f7536b9 --- /dev/null +++ b/policy/internal/yaml_string_element_scanner.h @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_YAML_STRING_ELEMENT_SCANNER_H_ +#define THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_YAML_STRING_ELEMENT_SCANNER_H_ + +#include + +#include "absl/strings/string_view.h" +#include "common/source.h" + +namespace cel::policy_internal { + +struct YamlStringElement { + SourcePosition starting_position = -1; + std::optional source_range; + bool quoted = false; +}; + +// Scans a YAML scalar string element directly from the SourceContentView +// (behaving as an array of char32_t unicode codepoints) starting at `pos`, +// matching against the decoded value `val`. +YamlStringElement ScanYamlStringElement(const SourceContentView& view, + SourcePosition pos, + absl::string_view val); + +} // namespace cel::policy_internal + +#endif // THIRD_PARTY_CEL_CPP_POLICY_INTERNAL_YAML_STRING_ELEMENT_SCANNER_H_ diff --git a/policy/internal/yaml_string_element_scanner_test.cc b/policy/internal/yaml_string_element_scanner_test.cc new file mode 100644 index 000000000..94cab4ba8 --- /dev/null +++ b/policy/internal/yaml_string_element_scanner_test.cc @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "policy/internal/yaml_string_element_scanner.h" + +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "internal/testing.h" + +namespace cel::policy_internal { +namespace { + +using ::testing::Eq; + +TEST(YamlStringElementScannerTest, QuotedScalars) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("expression: \"a + b\"")); + YamlStringElement element = + ScanYamlStringElement(source->content(), 12, "a + b"); + EXPECT_THAT(element.starting_position, Eq(12)); + EXPECT_THAT(element.quoted, Eq(true)); + ASSERT_THAT(element.source_range.has_value(), Eq(true)); + EXPECT_THAT(element.source_range->begin, Eq(13)); + EXPECT_THAT(element.source_range->end, Eq(18)); + + ASSERT_OK_AND_ASSIGN(auto source2, NewSource("expression: 'a + b'")); + YamlStringElement element2 = + ScanYamlStringElement(source2->content(), 12, "a + b"); + EXPECT_THAT(element2.starting_position, Eq(12)); + EXPECT_THAT(element2.quoted, Eq(true)); + ASSERT_THAT(element2.source_range.has_value(), Eq(true)); + EXPECT_THAT(element2.source_range->begin, Eq(13)); + EXPECT_THAT(element2.source_range->end, Eq(18)); + + ASSERT_OK_AND_ASSIGN(auto source3, NewSource("expression: \"a + \\n b\"")); + YamlStringElement element3 = + ScanYamlStringElement(source3->content(), 12, "a + \n b"); + EXPECT_THAT(element3.starting_position, Eq(12)); + EXPECT_THAT(element3.quoted, Eq(true)); + ASSERT_THAT(element3.source_range.has_value(), Eq(true)); + EXPECT_THAT(element3.source_range->begin, Eq(13)); + EXPECT_THAT(element3.source_range->end, Eq(21)); +} + +TEST(YamlStringElementScannerTest, PlainScalars) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("expression: a + b")); + YamlStringElement element = + ScanYamlStringElement(source->content(), 12, "a + b"); + EXPECT_THAT(element.starting_position, Eq(12)); + EXPECT_THAT(element.quoted, Eq(false)); + ASSERT_THAT(element.source_range.has_value(), Eq(true)); + EXPECT_THAT(element.source_range->begin, Eq(12)); + EXPECT_THAT(element.source_range->end, Eq(17)); + + ASSERT_OK_AND_ASSIGN(auto source2, NewSource("expression: a +\n b")); + YamlStringElement element2 = + ScanYamlStringElement(source2->content(), 12, "a + b"); + EXPECT_THAT(element2.starting_position, Eq(12)); + EXPECT_THAT(element2.quoted, Eq(false)); + ASSERT_THAT(element2.source_range.has_value(), Eq(true)); + EXPECT_THAT(element2.source_range->begin, Eq(12)); + EXPECT_THAT(element2.source_range->end, Eq(19)); +} + +TEST(YamlStringElementScannerTest, BlockScalars) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("expression: |\n a + b\n")); + YamlStringElement element = + ScanYamlStringElement(source->content(), 12, "a + b\n"); + EXPECT_THAT(element.starting_position, Eq(12)); + EXPECT_THAT(element.quoted, Eq(false)); + ASSERT_THAT(element.source_range.has_value(), Eq(true)); + EXPECT_THAT(element.source_range->begin, Eq(16)); + EXPECT_THAT(element.source_range->end, Eq(21)); + + ASSERT_OK_AND_ASSIGN(auto source2, NewSource("expression: >2-\n a + b\n")); + YamlStringElement element2 = + ScanYamlStringElement(source2->content(), 12, "a + b"); + EXPECT_THAT(element2.starting_position, Eq(12)); + EXPECT_THAT(element2.quoted, Eq(false)); + ASSERT_THAT(element2.source_range.has_value(), Eq(true)); + EXPECT_THAT(element2.source_range->begin, Eq(20)); + EXPECT_THAT(element2.source_range->end, Eq(25)); +} + +TEST(YamlStringElementScannerTest, InvalidPosition) { + ASSERT_OK_AND_ASSIGN(auto source, NewSource("expression: a + b")); + YamlStringElement element = + ScanYamlStringElement(source->content(), 100, "a + b"); + EXPECT_THAT(element.starting_position, Eq(100)); + EXPECT_THAT(element.quoted, Eq(false)); + EXPECT_THAT(element.source_range.has_value(), Eq(false)); +} + +} // namespace +} // namespace cel::policy_internal diff --git a/policy/yaml_policy_parser.cc b/policy/yaml_policy_parser.cc index c838cff33..4b4a37cb9 100644 --- a/policy/yaml_policy_parser.cc +++ b/policy/yaml_policy_parser.cc @@ -29,6 +29,7 @@ #include "policy/cel_policy_parse_context.h" #include "policy/cel_policy_parse_result.h" #include "policy/cel_policy_parser.h" +#include "policy/internal/yaml_string_element_scanner.h" #include "yaml-cpp/exceptions.h" #include "yaml-cpp/node/node.h" #include "yaml-cpp/node/parse.h" @@ -60,6 +61,17 @@ std::optional YamlPolicyParser::GetValueString( return std::nullopt; } + if (!node.Mark().is_null() && ctx.policy_source().content() != nullptr) { + policy_internal::YamlStringElement element = + policy_internal::ScanYamlStringElement( + ctx.policy_source().content()->content(), node.Mark().pos, + node.as()); + + ctx.policy_source().NoteSourcePosition(id, element.starting_position); + ctx.policy_source().NoteSourceRange(id, element.source_range, + element.quoted); + } + try { return ValueString(id, node.as()); } catch (YAML::Exception& e) { From a3fb693fefb424809c1030212f4a1aee95a5c17d Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 1 Jul 2026 15:02:25 -0700 Subject: [PATCH 099/350] Bump absl version and update CI - Update to latest absl release - Bump compiler versions in dockerfiles - update codelab reference dockerfile to better match the gcb image PiperOrigin-RevId: 941313594 --- Dockerfile | 69 ---------------------------------------------- MODULE.bazel | 2 +- cloudbuild.yaml | 17 ++++++++---- codelab/Dockerfile | 10 +++---- 4 files changed, 17 insertions(+), 81 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 97611fc75..000000000 --- a/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# This Dockerfile is used to create a container around gcc9 and bazel for -# building the CEL C++ library on GitHub. -# -# To update a new version of this container, use gcloud. You may need to run -# `gcloud auth login` and `gcloud auth configure-docker` first. -# -# Note, if you need to run docker using `sudo` use the following commands -# instead: -# -# sudo gcloud auth login --no-launch-browser -# sudo gcloud auth configure-docker -# -# Run the following command from the root of the CEL repository: -# -# gcloud builds submit --region=us -t gcr.io/cel-analysis/cel-cpp/ubuntu_floor . -# -# Once complete get the sha256 digest from the output using the following -# command: -# -# gcloud artifacts versions list --package=cel-cpp/ubuntu_floor --repository=gcr.io \ -# --location=us -# -# The cloudbuild.yaml file must be updated to use the new digest like so: -# -# - name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@' -FROM gcr.io/cloud-marketplace/google/ubuntu2204:latest - -# Install Bazel prerequesites and required tools. -# See https://docs.bazel.build/versions/master/install-ubuntu.html -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - bash \ - ca-certificates \ - git \ - libssl-dev \ - make \ - pkg-config \ - python3 \ - unzip \ - wget \ - zip \ - zlib1g-dev \ - default-jdk-headless \ - clang-11 \ - gcc-9 g++-9 \ - tzdata \ - && apt-get clean - -# Install Bazelisk. -# https://github.com/bazelbuild/bazelisk/releases -ARG BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/v1.27.0/bazelisk-amd64.deb" -ARG BAZELISK_CHKSUM="d8b00ea975c823e15263c80200ac42979e17368547fbff4ab177af035badfa83" -ADD ${BAZELISK_URL} /tmp/bazelisk.deb - -ENV BAZELISK_CHKSUM=${BAZELISK_CHKSUM} -RUN echo "${BAZELISK_CHKSUM} */tmp/bazelisk.deb" | sha256sum --check - -RUN apt-get install /tmp/bazelisk.deb - -RUN mkdir -p /workspace -RUN mkdir -p /bazel - -RUN USE_BAZEL_VERSION=8.7.0 bazelisk help -RUN USE_BAZEL_VERSION=7.3.2 bazelisk help - -ENV CC=gcc-9 -ENV CXX=g++-9 - -ENTRYPOINT ["/usr/bin/bazelisk"] diff --git a/MODULE.bazel b/MODULE.bazel index 187d68164..703063c8d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -39,7 +39,7 @@ bazel_dep( ) bazel_dep( name = "abseil-cpp", - version = "20260107.0", + version = "20260526.0", repo_name = "com_google_absl", ) bazel_dep( diff --git a/cloudbuild.yaml b/cloudbuild.yaml index dec359f25..fe78c7a2b 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,5 +1,5 @@ steps: -- name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@sha256:211a0c505b361d987b3d8b08a5144a84e62cb95edc3f897fe46d5cd3f556f79d' +- name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@sha256:fbcc9edb13aa855f435993bf12ec4fd59f8b72e7d715bbc4c72e7a2298f0bcc2' args: - '--output_base=/bazel' # This is mandatory to avoid steps accidently sharing data. - 'test' @@ -14,18 +14,23 @@ steps: - '--local_ram_resources=HOST_RAM*.4' - '--remote_cache=https://storage.googleapis.com/cel-cpp-remote-cache' - '--google_default_credentials' - id: gcc-9 + id: gcc-10 waitFor: ['-'] -- name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@sha256:211a0c505b361d987b3d8b08a5144a84e62cb95edc3f897fe46d5cd3f556f79d' +- name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@sha256:fbcc9edb13aa855f435993bf12ec4fd59f8b72e7d715bbc4c72e7a2298f0bcc2' env: - - 'CC=clang-11' - - 'CXX=clang++-11' + - 'CC=clang-14' + - 'CXX=clang++-14' args: - '--output_base=/bazel' # This is mandatory to avoid steps accidently sharing data. - 'test' - '...' - '--enable_bzlmod' - '--copt=-Wno-deprecated-declarations' + - '--copt=-Wno-unknown-warning-option' +# some older versions of clang (<16) complain about some template metaprogramming in +# absl in this release but otherwise it builds fine. Should be fixed in the next release. +# https://github.com/abseil/abseil-cpp/issues/2073 + - '--copt=-Wno-invalid-partial-specialization' - '--compilation_mode=fastbuild' - '--test_output=errors' - '--show_timestamps' @@ -34,7 +39,7 @@ steps: - '--local_ram_resources=HOST_RAM*.4' - '--remote_cache=https://storage.googleapis.com/cel-cpp-remote-cache' - '--google_default_credentials' - id: clang-11 + id: clang-14 waitFor: ['-'] timeout: 1h options: diff --git a/codelab/Dockerfile b/codelab/Dockerfile index c98a08f39..19040f2de 100644 --- a/codelab/Dockerfile +++ b/codelab/Dockerfile @@ -1,17 +1,17 @@ -ARG DEBIAN_IMAGE="marketplace.gcr.io/google/debian11:latest" -FROM ${DEBIAN_IMAGE} +ARG UBUNTU_IMAGE="ubuntu:22.04" +FROM ${UBUNTU_IMAGE} ARG BAZELISK_RELEASE="https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-amd64.deb" -RUN apt update && apt upgrade -y && apt install -y gcc-9 g++-9 clang-13 git curl bash openjdk-11-jdk-headless +RUN apt update && apt upgrade -y && apt install -y gcc-10 g++-10 clang-14 git curl bash openjdk-11-jdk-headless RUN curl -L ${BAZELISK_RELEASE} > ./bazelisk.deb RUN apt install ./bazelisk.deb RUN git clone https://github.com/google/cel-cpp.git -ENV CXX=clang++-13 -ENV CC=clang-13 +ENV CXX=g++-10 +ENV CC=gcc-10 WORKDIR /cel-cpp # not generally recommended to cache the bazel build in the image, From 8b547a20723ff32ddba4db24c1debcd256998f5c Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:08:40 -0700 Subject: [PATCH 100/350] No public description PiperOrigin-RevId: 942894276 --- eval/public/builtin_func_test.cc | 14 +++++++------- eval/public/cel_function_registry_test.cc | 2 +- eval/public/cel_number.cc | 2 +- eval/public/cel_number_test.cc | 2 +- eval/public/cel_type_registry.cc | 2 +- eval/public/cel_type_registry_test.cc | 2 +- eval/public/cel_value.cc | 2 +- eval/public/cel_value_test.cc | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/eval/public/builtin_func_test.cc b/eval/public/builtin_func_test.cc index b73a2dc55..a69091de4 100644 --- a/eval/public/builtin_func_test.cc +++ b/eval/public/builtin_func_test.cc @@ -1132,7 +1132,7 @@ class FakeErrorMap : public CelMap { } absl::optional operator[](CelValue key) const override { - return absl::nullopt; + return std::nullopt; } absl::StatusOr ListKeys() const override { @@ -1160,11 +1160,11 @@ class FakeMap : public CelMap { absl::optional operator[](CelValue key) const override { absl::optional raw_value = get_cel_value_(key); if (!raw_value) { - return absl::nullopt; + return std::nullopt; } auto it = data_.find(*raw_value); if (it == data_.end()) { - return absl::nullopt; + return std::nullopt; } return it->second; } @@ -1185,7 +1185,7 @@ class FakeBoolMap : public FakeMap { : FakeMap(data, CelValue::CreateBool, [](CelValue v) -> absl::optional { if (!v.IsBool()) { - return absl::nullopt; + return std::nullopt; } return v.BoolOrDie(); }) {} @@ -1197,7 +1197,7 @@ class FakeInt64Map : public FakeMap { : FakeMap(data, CelValue::CreateInt64, [](CelValue v) -> absl::optional { if (!v.IsInt64()) { - return absl::nullopt; + return std::nullopt; } return v.Int64OrDie(); }) {} @@ -1209,7 +1209,7 @@ class FakeUint64Map : public FakeMap { : FakeMap(data, CelValue::CreateUint64, [](CelValue v) -> absl::optional { if (!v.IsUint64()) { - return absl::nullopt; + return std::nullopt; } return v.Uint64OrDie(); }) {} @@ -1223,7 +1223,7 @@ class FakeStringMap : public FakeMap { [](CelValue::StringHolder v) { return CelValue::CreateString(v); }, [](CelValue v) -> absl::optional { if (!v.IsString()) { - return absl::nullopt; + return std::nullopt; } return v.StringOrDie(); }) {} diff --git a/eval/public/cel_function_registry_test.cc b/eval/public/cel_function_registry_test.cc index 75963cda7..7920d794f 100644 --- a/eval/public/cel_function_registry_test.cc +++ b/eval/public/cel_function_registry_test.cc @@ -151,7 +151,7 @@ TEST(CelFunctionRegistryTest, DefaultLazyProviderNoOverloadFound) { activation); ASSERT_OK(func.status()); - EXPECT_EQ(*func, absl::nullopt); + EXPECT_EQ(*func, std::nullopt); } TEST(CelFunctionRegistryTest, DefaultLazyProviderAmbiguousLookup) { diff --git a/eval/public/cel_number.cc b/eval/public/cel_number.cc index e08afb6a3..052a4f4e6 100644 --- a/eval/public/cel_number.cc +++ b/eval/public/cel_number.cc @@ -26,6 +26,6 @@ absl::optional GetNumberFromCelValue(const CelValue& value) { } else if (double val; value.GetValue(&val)) { return CelNumber(val); } - return absl::nullopt; + return std::nullopt; } } // namespace google::api::expr::runtime diff --git a/eval/public/cel_number_test.cc b/eval/public/cel_number_test.cc index 3c6f36e9b..bdddeaf20 100644 --- a/eval/public/cel_number_test.cc +++ b/eval/public/cel_number_test.cc @@ -36,7 +36,7 @@ TEST(CelNumber, GetNumberFromCelValue) { Optional(CelNumber::FromDouble(1.0))); EXPECT_EQ(GetNumberFromCelValue(CelValue::CreateDuration(absl::Seconds(1))), - absl::nullopt); + std::nullopt); } diff --git a/eval/public/cel_type_registry.cc b/eval/public/cel_type_registry.cc index 639a348dd..d43fddd16 100644 --- a/eval/public/cel_type_registry.cc +++ b/eval/public/cel_type_registry.cc @@ -58,7 +58,7 @@ absl::optional CelTypeRegistry::FindTypeAdapter( if (maybe_adapter.has_value()) { return maybe_adapter; } - return absl::nullopt; + return std::nullopt; } } // namespace google::api::expr::runtime diff --git a/eval/public/cel_type_registry_test.cc b/eval/public/cel_type_registry_test.cc index 9f3fde9be..881700d55 100644 --- a/eval/public/cel_type_registry_test.cc +++ b/eval/public/cel_type_registry_test.cc @@ -39,7 +39,7 @@ class TestTypeProvider : public LegacyTypeProvider { return LegacyTypeAdapter(/*access=*/nullptr, /*mutation=*/nullptr); } } - return absl::nullopt; + return std::nullopt; } private: diff --git a/eval/public/cel_value.cc b/eval/public/cel_value.cc index 25da7fe75..7e2d3b5d4 100644 --- a/eval/public/cel_value.cc +++ b/eval/public/cel_value.cc @@ -263,7 +263,7 @@ class EmptyCelMap final : public CelMap { } absl::optional operator[](CelValue key) const override { - return absl::nullopt; + return std::nullopt; } absl::StatusOr Has(const CelValue& key) const override { diff --git a/eval/public/cel_value_test.cc b/eval/public/cel_value_test.cc index 0af6eb9e7..985bb08a9 100644 --- a/eval/public/cel_value_test.cc +++ b/eval/public/cel_value_test.cc @@ -294,7 +294,7 @@ TEST(CelValueTest, TestEmptyMap) { EXPECT_EQ(value2->size(), 0); EXPECT_THAT(value2->Has(CelValue::CreateBool(false)), IsOkAndHolds(false)); EXPECT_THAT(value2->Get(&arena, CelValue::CreateBool(false)), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(value2->ListKeys(&arena), IsOkAndHolds(NotNull())); } From e246fa50b82b068aebdd2f3c139c915d5470931c Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:08:41 -0700 Subject: [PATCH 101/350] No public description PiperOrigin-RevId: 942894284 --- common/values/bytes_value_test.cc | 2 +- common/values/custom_list_value_test.cc | 8 +- common/values/custom_map_value.cc | 4 +- common/values/custom_map_value_test.cc | 12 +- common/values/legacy_list_value.cc | 2 +- common/values/legacy_map_value.cc | 2 +- common/values/legacy_struct_value.cc | 2 +- common/values/list_value.cc | 4 +- common/values/map_value.cc | 4 +- common/values/message_value.cc | 4 +- common/values/null_value_test.cc | 2 +- common/values/opaque_value.cc | 4 +- common/values/parsed_json_list_value.cc | 2 +- common/values/parsed_json_list_value_test.cc | 4 +- common/values/parsed_json_map_value_test.cc | 8 +- common/values/parsed_map_field_value.cc | 8 +- common/values/parsed_map_field_value_test.cc | 8 +- .../parsed_repeated_field_value_test.cc | 4 +- common/values/string_value.cc | 36 ++-- common/values/string_value_test.cc | 19 +- common/values/struct_value.cc | 8 +- common/values/struct_value_builder.cc | 168 +++++++++--------- common/values/value_builder.cc | 8 +- 23 files changed, 161 insertions(+), 162 deletions(-) diff --git a/common/values/bytes_value_test.cc b/common/values/bytes_value_test.cc index 58219e3a4..367a8ca16 100644 --- a/common/values/bytes_value_test.cc +++ b/common/values/bytes_value_test.cc @@ -80,7 +80,7 @@ TEST_F(BytesValueTest, TryFlat) { EXPECT_THAT( BytesValue(absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"})) .TryFlat(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } TEST_F(BytesValueTest, ToString) { diff --git a/common/values/custom_list_value_test.cc b/common/values/custom_list_value_test.cc index 79c3f2419..ea0c53d2e 100644 --- a/common/values/custom_list_value_test.cc +++ b/common/values/custom_list_value_test.cc @@ -442,7 +442,7 @@ TEST_F(CustomListValueTest, Dispatcher_NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(IntValueIs(1)))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomListValueTest, Interface_NewIterator1) { @@ -453,7 +453,7 @@ TEST_F(CustomListValueTest, Interface_NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(IntValueIs(1)))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomListValueTest, Dispatcher_NewIterator2) { @@ -464,7 +464,7 @@ TEST_F(CustomListValueTest, Dispatcher_NewIterator2) { EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(IntValueIs(1), IntValueIs(1))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomListValueTest, Interface_NewIterator2) { @@ -475,7 +475,7 @@ TEST_F(CustomListValueTest, Interface_NewIterator2) { EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(IntValueIs(1), IntValueIs(1))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomListValueTest, Dispatcher_Contains) { diff --git a/common/values/custom_map_value.cc b/common/values/custom_map_value.cc index ae07f7723..495b64b7a 100644 --- a/common/values/custom_map_value.cc +++ b/common/values/custom_map_value.cc @@ -106,13 +106,13 @@ class EmptyMapValue final : public common_internal::CompatMapValue { } absl::optional operator[](CelValue key) const override { - return absl::nullopt; + return std::nullopt; } using CompatMapValue::Get; absl::optional Get(google::protobuf::Arena* arena, CelValue key) const override { - return absl::nullopt; + return std::nullopt; } absl::StatusOr Has(const CelValue& key) const override { return false; } diff --git a/common/values/custom_map_value_test.cc b/common/values/custom_map_value_test.cc index 8c3183cf8..d19bca373 100644 --- a/common/values/custom_map_value_test.cc +++ b/common/values/custom_map_value_test.cc @@ -480,7 +480,7 @@ TEST_F(CustomMapValueTest, Dispatcher_Find) { IsOkAndHolds(Optional(IntValueIs(1)))); ASSERT_THAT(map.Find(StringValue("baz"), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Interface_Find) { @@ -493,7 +493,7 @@ TEST_F(CustomMapValueTest, Interface_Find) { IsOkAndHolds(Optional(IntValueIs(1)))); ASSERT_THAT(map.Find(StringValue("baz"), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Dispatcher_Has) { @@ -588,7 +588,7 @@ TEST_F(CustomMapValueTest, Dispatcher_NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(StringValueIs("bar")))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Interface_NewIterator1) { @@ -599,7 +599,7 @@ TEST_F(CustomMapValueTest, Interface_NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(StringValueIs("bar")))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Dispatcher_NewIterator2) { @@ -612,7 +612,7 @@ TEST_F(CustomMapValueTest, Dispatcher_NewIterator2) { iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(StringValueIs("bar"), IntValueIs(1))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Interface_NewIterator2) { @@ -625,7 +625,7 @@ TEST_F(CustomMapValueTest, Interface_NewIterator2) { iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(StringValueIs("bar"), IntValueIs(1))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Dispatcher) { diff --git a/common/values/legacy_list_value.cc b/common/values/legacy_list_value.cc index 93848ca44..60e571452 100644 --- a/common/values/legacy_list_value.cc +++ b/common/values/legacy_list_value.cc @@ -70,7 +70,7 @@ absl::optional AsLegacyListValue(const Value& value) { custom_list_value->interface()))); } } - return absl::nullopt; + return std::nullopt; } } // namespace cel::common_internal diff --git a/common/values/legacy_map_value.cc b/common/values/legacy_map_value.cc index 1f370761e..5b24b34ba 100644 --- a/common/values/legacy_map_value.cc +++ b/common/values/legacy_map_value.cc @@ -70,7 +70,7 @@ absl::optional AsLegacyMapValue(const Value& value) { custom_map_value->interface()))); } } - return absl::nullopt; + return std::nullopt; } } // namespace cel::common_internal diff --git a/common/values/legacy_struct_value.cc b/common/values/legacy_struct_value.cc index 4a91c5d42..aa342ad06 100644 --- a/common/values/legacy_struct_value.cc +++ b/common/values/legacy_struct_value.cc @@ -37,7 +37,7 @@ absl::optional AsLegacyStructValue(const Value& value) { if (IsLegacyStructValue(value)) { return GetLegacyStructValue(value); } - return absl::nullopt; + return std::nullopt; } } // namespace cel::common_internal diff --git a/common/values/list_value.cc b/common/values/list_value.cc index 35df98c40..6dad0ad94 100644 --- a/common/values/list_value.cc +++ b/common/values/list_value.cc @@ -263,7 +263,7 @@ optional_ref ListValue::AsCustom() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional ListValue::AsCustom() && { @@ -271,7 +271,7 @@ absl::optional ListValue::AsCustom() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } const CustomListValue& ListValue::GetCustom() const& { diff --git a/common/values/map_value.cc b/common/values/map_value.cc index c8bf7b785..46891d1ce 100644 --- a/common/values/map_value.cc +++ b/common/values/map_value.cc @@ -337,7 +337,7 @@ optional_ref MapValue::AsCustom() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional MapValue::AsCustom() && { @@ -345,7 +345,7 @@ absl::optional MapValue::AsCustom() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } const CustomMapValue& MapValue::GetCustom() const& { diff --git a/common/values/message_value.cc b/common/values/message_value.cc index 66dfd9511..028641dbe 100644 --- a/common/values/message_value.cc +++ b/common/values/message_value.cc @@ -262,7 +262,7 @@ cel::optional_ref MessageValue::AsParsed() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional MessageValue::AsParsed() && { @@ -270,7 +270,7 @@ absl::optional MessageValue::AsParsed() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } const ParsedMessageValue& MessageValue::GetParsed() const& { diff --git a/common/values/null_value_test.cc b/common/values/null_value_test.cc index 5f244c532..f217ddac4 100644 --- a/common/values/null_value_test.cc +++ b/common/values/null_value_test.cc @@ -75,7 +75,7 @@ TEST_F(NullValueTest, Cast) { } TEST_F(NullValueTest, As) { - EXPECT_THAT(As(Value(NullValue())), Ne(absl::nullopt)); + EXPECT_THAT(As(Value(NullValue())), Ne(std::nullopt)); } } // namespace diff --git a/common/values/opaque_value.cc b/common/values/opaque_value.cc index 235d268e7..4788abed0 100644 --- a/common/values/opaque_value.cc +++ b/common/values/opaque_value.cc @@ -171,14 +171,14 @@ optional_ref OpaqueValue::AsOptional() const& { if (IsOptional()) { return *reinterpret_cast(this); } - return absl::nullopt; + return std::nullopt; } absl::optional OpaqueValue::AsOptional() && { if (IsOptional()) { return std::move(*reinterpret_cast(this)); } - return absl::nullopt; + return std::nullopt; } const OptionalValue& OpaqueValue::GetOptional() const& { diff --git a/common/values/parsed_json_list_value.cc b/common/values/parsed_json_list_value.cc index 9acd23e3f..936853f23 100644 --- a/common/values/parsed_json_list_value.cc +++ b/common/values/parsed_json_list_value.cc @@ -365,7 +365,7 @@ absl::optional AsNumber(const Value& value) { if (auto double_value = value.AsDouble(); double_value) { return internal::Number::FromDouble(*double_value); } - return absl::nullopt; + return std::nullopt; } } // namespace diff --git a/common/values/parsed_json_list_value_test.cc b/common/values/parsed_json_list_value_test.cc index 017a24f9d..a7ce17410 100644 --- a/common/values/parsed_json_list_value_test.cc +++ b/common/values/parsed_json_list_value_test.cc @@ -203,7 +203,7 @@ TEST_F(ParsedJsonListValueTest, NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedJsonListValueTest, NewIterator2) { @@ -218,7 +218,7 @@ TEST_F(ParsedJsonListValueTest, NewIterator2) { EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(IntValueIs(1), BoolValueIs(true))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedJsonListValueTest, Contains_Dynamic) { diff --git a/common/values/parsed_json_map_value_test.cc b/common/values/parsed_json_map_value_test.cc index b65128076..acc29deb2 100644 --- a/common/values/parsed_json_map_value_test.cc +++ b/common/values/parsed_json_map_value_test.cc @@ -170,7 +170,7 @@ TEST_F(ParsedJsonMapValueTest, Find_Dynamic) { arena()); EXPECT_THAT(valid_value.Find(BoolValue(), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(valid_value.Find(StringValue("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(IsNullValue()))); @@ -179,7 +179,7 @@ TEST_F(ParsedJsonMapValueTest, Find_Dynamic) { IsOkAndHolds(Optional(BoolValueIs(true)))); EXPECT_THAT(valid_value.Find(StringValue("baz"), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedJsonMapValueTest, Has_Dynamic) { @@ -308,7 +308,7 @@ TEST_F(ParsedJsonMapValueTest, NewIterator1) { IsOkAndHolds( Optional(AnyOf(StringValueIs("foo"), StringValueIs("bar"))))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedJsonMapValueTest, NewIterator2) { @@ -333,7 +333,7 @@ TEST_F(ParsedJsonMapValueTest, NewIterator2) { AnyOf(Pair(StringValueIs("foo"), IsNullValue()), Pair(StringValueIs("bar"), BoolValueIs(true)))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } } // namespace diff --git a/common/values/parsed_map_field_value.cc b/common/values/parsed_map_field_value.cc index 47b737f82..97450aea0 100644 --- a/common/values/parsed_map_field_value.cc +++ b/common/values/parsed_map_field_value.cc @@ -203,7 +203,7 @@ absl::optional ValueAsInt32(const Value& value) { double_value->NativeValue())) == double_value->NativeValue()) { return static_cast(double_value->NativeValue()); } - return absl::nullopt; + return std::nullopt; } absl::optional ValueAsInt64(const Value& value) { @@ -219,7 +219,7 @@ absl::optional ValueAsInt64(const Value& value) { double_value->NativeValue())) == double_value->NativeValue()) { return static_cast(double_value->NativeValue()); } - return absl::nullopt; + return std::nullopt; } absl::optional ValueAsUInt32(const Value& value) { @@ -237,7 +237,7 @@ absl::optional ValueAsUInt32(const Value& value) { double_value->NativeValue())) == double_value->NativeValue()) { return static_cast(double_value->NativeValue()); } - return absl::nullopt; + return std::nullopt; } absl::optional ValueAsUInt64(const Value& value) { @@ -252,7 +252,7 @@ absl::optional ValueAsUInt64(const Value& value) { double_value->NativeValue())) == double_value->NativeValue()) { return static_cast(double_value->NativeValue()); } - return absl::nullopt; + return std::nullopt; } bool ValueToProtoMapKey(const Value& key, diff --git a/common/values/parsed_map_field_value_test.cc b/common/values/parsed_map_field_value_test.cc index 271813f40..93c59a98f 100644 --- a/common/values/parsed_map_field_value_test.cc +++ b/common/values/parsed_map_field_value_test.cc @@ -213,7 +213,7 @@ TEST_F(ParsedMapFieldValueTest, Find) { DynamicGetField("map_string_bool"), arena()); EXPECT_THAT( value.Find(BoolValue(), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(value.Find(StringValue("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(false)))); @@ -222,7 +222,7 @@ TEST_F(ParsedMapFieldValueTest, Find) { IsOkAndHolds(Optional(BoolValueIs(true)))); EXPECT_THAT(value.Find(StringValue("baz"), descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedMapFieldValueTest, Has) { @@ -544,7 +544,7 @@ TEST_F(ParsedMapFieldValueTest, NewIterator1) { IsOkAndHolds( Optional(AnyOf(StringValueIs("foo"), StringValueIs("bar"))))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedMapFieldValueTest, NewIterator2) { @@ -564,7 +564,7 @@ TEST_F(ParsedMapFieldValueTest, NewIterator2) { AnyOf(Pair(StringValueIs("foo"), BoolValueIs(false)), Pair(StringValueIs("bar"), BoolValueIs(true)))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } } // namespace diff --git a/common/values/parsed_repeated_field_value_test.cc b/common/values/parsed_repeated_field_value_test.cc index 3155e7159..6941de0e9 100644 --- a/common/values/parsed_repeated_field_value_test.cc +++ b/common/values/parsed_repeated_field_value_test.cc @@ -396,7 +396,7 @@ TEST_F(ParsedRepeatedFieldValueTest, NewIterator1) { EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedRepeatedFieldValueTest, NewIterator2) { @@ -410,7 +410,7 @@ TEST_F(ParsedRepeatedFieldValueTest, NewIterator2) { EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(Pair(IntValueIs(1), BoolValueIs(true))))); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ParsedRepeatedFieldValueTest, Contains) { diff --git a/common/values/string_value.cc b/common/values/string_value.cc index 59662921a..9ebde8795 100644 --- a/common/values/string_value.cc +++ b/common/values/string_value.cc @@ -246,7 +246,7 @@ absl::optional StringValue::IndexOf(absl::string_view string) const { lhs.remove_prefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; }, [&](absl::Cord lhs) -> absl::optional { int64_t code_points = 0; @@ -262,7 +262,7 @@ absl::optional StringValue::IndexOf(absl::string_view string) const { lhs.RemovePrefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; })); } @@ -282,7 +282,7 @@ absl::optional StringValue::IndexOf(const absl::Cord& string) const { lhs.remove_prefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; }, [&](absl::Cord lhs) -> absl::optional { int64_t code_points = 0; @@ -298,7 +298,7 @@ absl::optional StringValue::IndexOf(const absl::Cord& string) const { lhs.RemovePrefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; })); } @@ -332,7 +332,7 @@ absl::optional StringValue::IndexOf(absl::string_view string, lhs.remove_prefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; }, [&](absl::Cord lhs) -> absl::optional { int64_t code_points = 0; @@ -348,7 +348,7 @@ absl::optional StringValue::IndexOf(absl::string_view string, lhs.RemovePrefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; })); } @@ -372,7 +372,7 @@ absl::optional StringValue::IndexOf(const absl::Cord& string, lhs.remove_prefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; }, [&](absl::Cord lhs) -> absl::optional { int64_t code_points = 0; @@ -388,7 +388,7 @@ absl::optional StringValue::IndexOf(const absl::Cord& string, lhs.RemovePrefix(code_units); ++code_points; } - return absl::nullopt; + return std::nullopt; })); } @@ -421,7 +421,7 @@ absl::optional StringValue::LastIndexOf( lhs.remove_prefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; }, [&](absl::Cord lhs) -> absl::optional { @@ -439,7 +439,7 @@ absl::optional StringValue::LastIndexOf( lhs.RemovePrefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; })); } @@ -462,7 +462,7 @@ absl::optional StringValue::LastIndexOf( lhs.remove_prefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; }, [&](absl::Cord lhs) -> absl::optional { @@ -480,7 +480,7 @@ absl::optional StringValue::LastIndexOf( lhs.RemovePrefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; })); } @@ -499,7 +499,7 @@ absl::optional StringValue::LastIndexOf( absl::optional StringValue::LastIndexOf(absl::string_view string, int64_t pos) const { if (pos < 0) { - return absl::nullopt; + return std::nullopt; } return value_.Visit(absl::Overload( [&](absl::string_view lhs) -> absl::optional { @@ -517,7 +517,7 @@ absl::optional StringValue::LastIndexOf(absl::string_view string, lhs.remove_prefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; }, [&](absl::Cord lhs) -> absl::optional { @@ -535,7 +535,7 @@ absl::optional StringValue::LastIndexOf(absl::string_view string, lhs.RemovePrefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; })); } @@ -543,7 +543,7 @@ absl::optional StringValue::LastIndexOf(absl::string_view string, absl::optional StringValue::LastIndexOf(const absl::Cord& string, int64_t pos) const { if (pos < 0) { - return absl::nullopt; + return std::nullopt; } return value_.Visit(absl::Overload( [&](absl::string_view lhs) -> absl::optional { @@ -561,7 +561,7 @@ absl::optional StringValue::LastIndexOf(const absl::Cord& string, lhs.remove_prefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; }, [&](absl::Cord lhs) -> absl::optional { @@ -579,7 +579,7 @@ absl::optional StringValue::LastIndexOf(const absl::Cord& string, lhs.RemovePrefix(code_units); ++code_points; } - if (last_index < 0) return absl::nullopt; + if (last_index < 0) return std::nullopt; return last_index; })); } diff --git a/common/values/string_value_test.cc b/common/values/string_value_test.cc index 380746e1e..b4fa404ae 100644 --- a/common/values/string_value_test.cc +++ b/common/values/string_value_test.cc @@ -82,7 +82,7 @@ TEST_F(StringValueTest, TryFlat) { EXPECT_THAT( StringValue(absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"})) .TryFlat(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } TEST_F(StringValueTest, ToString) { @@ -227,7 +227,7 @@ TEST_F(StringValueTest, IndexOf) { EXPECT_THAT(big_string.IndexOf("is"), Optional(Eq(2))); EXPECT_THAT(big_string_cord.IndexOf("is"), Optional(Eq(2))); - EXPECT_THAT(big_string_cord.IndexOf("not found"), Eq(absl::nullopt)); + EXPECT_THAT(big_string_cord.IndexOf("not found"), Eq(std::nullopt)); EXPECT_THAT(big_string.IndexOf(small_string, 4), Optional(Eq(12))); EXPECT_THAT(big_string.IndexOf(small_string_cord, 4), Optional(Eq(12))); @@ -237,16 +237,15 @@ TEST_F(StringValueTest, IndexOf) { EXPECT_THAT(big_string.IndexOf("is", 4), Optional(Eq(12))); EXPECT_THAT(big_string_cord.IndexOf("is", 4), Optional(Eq(12))); - EXPECT_THAT(big_string.IndexOf(small_string, 13), Eq(absl::nullopt)); - EXPECT_THAT(big_string.IndexOf(small_string_cord, 13), Eq(absl::nullopt)); - EXPECT_THAT(big_string_cord.IndexOf(small_string, 13), Eq(absl::nullopt)); - EXPECT_THAT(big_string_cord.IndexOf(small_string_cord, 13), - Eq(absl::nullopt)); + EXPECT_THAT(big_string.IndexOf(small_string, 13), Eq(std::nullopt)); + EXPECT_THAT(big_string.IndexOf(small_string_cord, 13), Eq(std::nullopt)); + EXPECT_THAT(big_string_cord.IndexOf(small_string, 13), Eq(std::nullopt)); + EXPECT_THAT(big_string_cord.IndexOf(small_string_cord, 13), Eq(std::nullopt)); EXPECT_THAT(big_string.IndexOf(absl::Cord("is"), 4), Optional(Eq(12))); EXPECT_THAT(big_string_cord.IndexOf(absl::Cord("is"), 4), Optional(Eq(12))); - EXPECT_THAT(big_string.IndexOf(absl::Cord("is"), 13), Eq(absl::nullopt)); - EXPECT_THAT(big_string_cord.IndexOf(absl::Cord("is"), 13), Eq(absl::nullopt)); + EXPECT_THAT(big_string.IndexOf(absl::Cord("is"), 13), Eq(std::nullopt)); + EXPECT_THAT(big_string_cord.IndexOf(absl::Cord("is"), 13), Eq(std::nullopt)); } TEST_F(StringValueTest, LowerAscii) { @@ -318,7 +317,7 @@ TEST_F(StringValueTest, LastIndexOf) { EXPECT_THAT(big_string.LastIndexOf("is"), Optional(Eq(12))); EXPECT_THAT(big_string_cord.LastIndexOf("is"), Optional(Eq(12))); - EXPECT_THAT(big_string_cord.LastIndexOf("not found"), Eq(absl::nullopt)); + EXPECT_THAT(big_string_cord.LastIndexOf("not found"), Eq(std::nullopt)); EXPECT_THAT(big_string.LastIndexOf(small_string, 4), Optional(Eq(2))); EXPECT_THAT(big_string.LastIndexOf(small_string_cord, 4), Optional(Eq(2))); diff --git a/common/values/struct_value.cc b/common/values/struct_value.cc index 10238a670..efbcaa514 100644 --- a/common/values/struct_value.cc +++ b/common/values/struct_value.cc @@ -321,7 +321,7 @@ absl::optional StructValue::AsMessage() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional StructValue::AsMessage() && { @@ -329,7 +329,7 @@ absl::optional StructValue::AsMessage() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref StructValue::AsParsedMessage() const& { @@ -337,7 +337,7 @@ optional_ref StructValue::AsParsedMessage() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional StructValue::AsParsedMessage() && { @@ -345,7 +345,7 @@ absl::optional StructValue::AsParsedMessage() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } MessageValue StructValue::GetMessage() const& { diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index 446b18421..d5206acbf 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -75,7 +75,7 @@ absl::StatusOr> ProtoMessageCopyUsingSerialization( return absl::UnknownError( absl::StrCat("failed to parse `", to->GetTypeName(), "`")); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> ProtoMessageCopy( @@ -87,7 +87,7 @@ absl::StatusOr> ProtoMessageCopy( if (to_descriptor == from_descriptor) { // Same. to_message->CopyFrom(*from_message); - return absl::nullopt; + return std::nullopt; } if (to_descriptor->full_name() == from_descriptor->full_name()) { // Same type, different descriptors. @@ -110,7 +110,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->FloatValue().SetValue( message, static_cast(double_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -120,7 +120,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->DoubleValue().SetValue(message, double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -134,7 +134,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->Int32Value().SetValue( message, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -144,7 +144,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->Int64Value().SetValue(message, int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -157,7 +157,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->UInt32Value().SetValue( message, static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -167,7 +167,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->UInt64Value().SetValue(message, uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -177,7 +177,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->StringValue().SetValue(message, string_value->NativeCord()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -187,7 +187,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( message->GetDescriptor())); well_known_types->BytesValue().SetValue(message, bytes_value->NativeCord()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -197,7 +197,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( well_known_types->BoolValue().Initialize(message->GetDescriptor())); well_known_types->BoolValue().SetValue(message, bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -248,7 +248,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( well_known_types->Any().SetTypeUrl(message, type_url); well_known_types->Any().SetValue(message, std::move(serialized).Consume()); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: { if (auto duration_value = value.AsDuration(); duration_value) { @@ -256,7 +256,7 @@ absl::StatusOr> ProtoMessageFromValueImpl( well_known_types->Duration().Initialize(message->GetDescriptor())); CEL_RETURN_IF_ERROR(well_known_types->Duration().SetFromAbslDuration( message, duration_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } @@ -266,21 +266,21 @@ absl::StatusOr> ProtoMessageFromValueImpl( well_known_types->Timestamp().Initialize(message->GetDescriptor())); CEL_RETURN_IF_ERROR(well_known_types->Timestamp().SetFromAbslTime( message, timestamp_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); } case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: { CEL_RETURN_IF_ERROR(value.ConvertToJson(pool, factory, message)); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE: { CEL_RETURN_IF_ERROR(value.ConvertToJsonArray(pool, factory, message)); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT: { CEL_RETURN_IF_ERROR(value.ConvertToJsonObject(pool, factory, message)); - return absl::nullopt; + return std::nullopt; } default: break; @@ -315,7 +315,7 @@ absl::StatusOr> ProtoBoolMapKeyFromValueConverter( const Value& value, google::protobuf::MapKey& key, std::string&) { if (auto bool_value = value.AsBool(); bool_value) { key.SetBoolValue(bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "bool"); } @@ -328,7 +328,7 @@ absl::StatusOr> ProtoInt32MapKeyFromValueConverter( return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); } key.SetInt32Value(static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "int"); } @@ -337,7 +337,7 @@ absl::StatusOr> ProtoInt64MapKeyFromValueConverter( const Value& value, google::protobuf::MapKey& key, std::string&) { if (auto int_value = value.AsInt(); int_value) { key.SetInt64Value(int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "int"); } @@ -349,7 +349,7 @@ absl::StatusOr> ProtoUInt32MapKeyFromValueConverter( return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); } key.SetUInt32Value(static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "uint"); } @@ -358,7 +358,7 @@ absl::StatusOr> ProtoUInt64MapKeyFromValueConverter( const Value& value, google::protobuf::MapKey& key, std::string&) { if (auto uint_value = value.AsUint(); uint_value) { key.SetUInt64Value(uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "uint"); } @@ -368,7 +368,7 @@ absl::StatusOr> ProtoStringMapKeyFromValueConverter( if (auto string_value = value.AsString(); string_value) { key_string = string_value->NativeString(); key.SetStringValue(key_string); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "string"); } @@ -412,7 +412,7 @@ absl::StatusOr> ProtoBoolMapValueFromValueConverter( google::protobuf::MapValueRef& value_ref) { if (auto bool_value = value.AsBool(); bool_value) { value_ref.SetBoolValue(bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "bool"); } @@ -429,7 +429,7 @@ absl::StatusOr> ProtoInt32MapValueFromValueConverter( return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); } value_ref.SetInt32Value(static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "int"); } @@ -442,7 +442,7 @@ absl::StatusOr> ProtoInt64MapValueFromValueConverter( google::protobuf::MapValueRef& value_ref) { if (auto int_value = value.AsInt(); int_value) { value_ref.SetInt64Value(int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "int"); } @@ -459,7 +459,7 @@ ProtoUInt32MapValueFromValueConverter( return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); } value_ref.SetUInt32Value(static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "uint"); } @@ -473,7 +473,7 @@ ProtoUInt64MapValueFromValueConverter( google::protobuf::MapValueRef& value_ref) { if (auto uint_value = value.AsUint(); uint_value) { value_ref.SetUInt64Value(uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "uint"); } @@ -486,7 +486,7 @@ absl::StatusOr> ProtoFloatMapValueFromValueConverter( google::protobuf::MapValueRef& value_ref) { if (auto double_value = value.AsDouble(); double_value) { value_ref.SetFloatValue(double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "double"); } @@ -500,7 +500,7 @@ ProtoDoubleMapValueFromValueConverter( google::protobuf::MapValueRef& value_ref) { if (auto double_value = value.AsDouble(); double_value) { value_ref.SetDoubleValue(double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "double"); } @@ -513,7 +513,7 @@ absl::StatusOr> ProtoBytesMapValueFromValueConverter( google::protobuf::MapValueRef& value_ref) { if (auto bytes_value = value.AsBytes(); bytes_value) { value_ref.SetStringValue(bytes_value->NativeString()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "bytes"); } @@ -527,7 +527,7 @@ ProtoStringMapValueFromValueConverter( google::protobuf::MapValueRef& value_ref) { if (auto string_value = value.AsString(); string_value) { value_ref.SetStringValue(string_value->NativeString()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "string"); } @@ -540,7 +540,7 @@ absl::StatusOr> ProtoNullMapValueFromValueConverter( google::protobuf::MapValueRef& value_ref) { if (value.IsNull() || value.IsInt()) { value_ref.SetEnumValue(0); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "google.protobuf.NullValue"); } @@ -557,7 +557,7 @@ absl::StatusOr> ProtoEnumMapValueFromValueConverter( return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); } value_ref.SetEnumValue(static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "enum"); } @@ -632,7 +632,7 @@ ProtoBoolRepeatedFieldFromValueMutator( const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { if (auto bool_value = value.AsBool(); bool_value) { reflection->AddBool(message, field, bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "bool"); } @@ -652,7 +652,7 @@ ProtoInt32RepeatedFieldFromValueMutator( } reflection->AddInt32(message, field, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "int"); } @@ -667,7 +667,7 @@ ProtoInt64RepeatedFieldFromValueMutator( const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { if (auto int_value = value.AsInt(); int_value) { reflection->AddInt64(message, field, int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "int"); } @@ -686,7 +686,7 @@ ProtoUInt32RepeatedFieldFromValueMutator( } reflection->AddUInt32(message, field, static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "uint"); } @@ -701,7 +701,7 @@ ProtoUInt64RepeatedFieldFromValueMutator( const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { if (auto uint_value = value.AsUint(); uint_value) { reflection->AddUInt64(message, field, uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "uint"); } @@ -717,7 +717,7 @@ ProtoFloatRepeatedFieldFromValueMutator( if (auto double_value = value.AsDouble(); double_value) { reflection->AddFloat(message, field, static_cast(double_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "double"); } @@ -732,7 +732,7 @@ ProtoDoubleRepeatedFieldFromValueMutator( const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { if (auto double_value = value.AsDouble(); double_value) { reflection->AddDouble(message, field, double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "double"); } @@ -747,7 +747,7 @@ ProtoBytesRepeatedFieldFromValueMutator( const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { if (auto bytes_value = value.AsBytes(); bytes_value) { reflection->AddString(message, field, bytes_value->NativeString()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "bytes"); } @@ -762,7 +762,7 @@ ProtoStringRepeatedFieldFromValueMutator( const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { if (auto string_value = value.AsString(); string_value) { reflection->AddString(message, field, string_value->NativeString()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "string"); } @@ -777,7 +777,7 @@ ProtoNullRepeatedFieldFromValueMutator( const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { if (value.IsNull() || value.IsInt()) { reflection->AddEnumValue(message, field, 0); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "null_type"); } @@ -799,7 +799,7 @@ ProtoEnumRepeatedFieldFromValueMutator( } reflection->AddEnumValue(message, field, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), enum_descriptor->full_name()); } @@ -820,7 +820,7 @@ ProtoMessageRepeatedFieldFromValueMutator( well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT && well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY) { - return absl::nullopt; + return std::nullopt; } } auto* element = reflection->AddMessage(message, field, factory); @@ -1028,7 +1028,7 @@ class MessageValueBuilderImpl { case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { if (auto bool_value = value.AsBool(); bool_value) { reflection_->SetBool(message_, field, bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "bool"); } @@ -1040,14 +1040,14 @@ class MessageValueBuilderImpl { } reflection_->SetInt32(message_, field, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "int"); } case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { if (auto int_value = value.AsInt(); int_value) { reflection_->SetInt64(message_, field, int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "int"); } @@ -1061,28 +1061,28 @@ class MessageValueBuilderImpl { reflection_->SetUInt32( message_, field, static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "uint"); } case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { if (auto uint_value = value.AsUint(); uint_value) { reflection_->SetUInt64(message_, field, uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "uint"); } case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { if (auto double_value = value.AsDouble(); double_value) { reflection_->SetFloat(message_, field, double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "double"); } case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { if (auto double_value = value.AsDouble(); double_value) { reflection_->SetDouble(message_, field, double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "double"); } @@ -1096,7 +1096,7 @@ class MessageValueBuilderImpl { [this, field](const absl::Cord& cord) { reflection_->SetString(message_, field, cord); })); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "bytes"); } @@ -1108,7 +1108,7 @@ class MessageValueBuilderImpl { [this, field](const absl::Cord& cord) { reflection_->SetString(message_, field, cord); })); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "string"); } @@ -1116,7 +1116,7 @@ class MessageValueBuilderImpl { if (field->enum_type()->full_name() == "google.protobuf.NullValue") { if (value.IsNull() || value.IsInt()) { reflection_->SetEnumValue(message_, field, 0); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), "null_type"); } @@ -1125,7 +1125,7 @@ class MessageValueBuilderImpl { int_value->NativeValue() <= std::numeric_limits::max()) { reflection_->SetEnumValue( message_, field, static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } } return TypeConversionError(value.GetTypeName(), @@ -1136,7 +1136,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_BOOLVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto bool_value = value.AsBool(); bool_value) { CEL_RETURN_IF_ERROR(well_known_types_.BoolValue().Initialize( @@ -1145,7 +1145,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), bool_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1153,7 +1153,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < @@ -1168,7 +1168,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), static_cast(int_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1176,7 +1176,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto int_value = value.AsInt(); int_value) { CEL_RETURN_IF_ERROR(well_known_types_.Int64Value().Initialize( @@ -1185,7 +1185,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), int_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1193,7 +1193,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > @@ -1206,7 +1206,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), static_cast(uint_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1214,7 +1214,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto uint_value = value.AsUint(); uint_value) { CEL_RETURN_IF_ERROR(well_known_types_.UInt64Value().Initialize( @@ -1223,7 +1223,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), uint_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1231,7 +1231,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto double_value = value.AsDouble(); double_value) { CEL_RETURN_IF_ERROR(well_known_types_.FloatValue().Initialize( @@ -1240,7 +1240,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), static_cast(double_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1248,7 +1248,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto double_value = value.AsDouble(); double_value) { CEL_RETURN_IF_ERROR(well_known_types_.DoubleValue().Initialize( @@ -1257,7 +1257,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), double_value->NativeValue()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1265,7 +1265,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto bytes_value = value.AsBytes(); bytes_value) { CEL_RETURN_IF_ERROR(well_known_types_.BytesValue().Initialize( @@ -1274,7 +1274,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), bytes_value->NativeCord()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1282,7 +1282,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto string_value = value.AsString(); string_value) { CEL_RETURN_IF_ERROR(well_known_types_.StringValue().Initialize( @@ -1291,7 +1291,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), string_value->NativeCord()); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1299,7 +1299,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto duration_value = value.AsDuration(); duration_value) { CEL_RETURN_IF_ERROR(well_known_types_.Duration().Initialize( @@ -1309,7 +1309,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), duration_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1317,7 +1317,7 @@ class MessageValueBuilderImpl { case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: { if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } if (auto timestamp_value = value.AsTimestamp(); timestamp_value) { CEL_RETURN_IF_ERROR(well_known_types_.Timestamp().Initialize( @@ -1326,7 +1326,7 @@ class MessageValueBuilderImpl { reflection_->MutableMessage(message_, field, message_factory_), timestamp_value->NativeValue())); - return absl::nullopt; + return std::nullopt; } return TypeConversionError(value.GetTypeName(), field->message_type()->full_name()); @@ -1336,21 +1336,21 @@ class MessageValueBuilderImpl { value.ConvertToJson(descriptor_pool_, message_factory_, reflection_->MutableMessage( message_, field, message_factory_))); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE: { CEL_RETURN_IF_ERROR(value.ConvertToJsonArray( descriptor_pool_, message_factory_, reflection_->MutableMessage(message_, field, message_factory_))); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT: { CEL_RETURN_IF_ERROR(value.ConvertToJsonObject( descriptor_pool_, message_factory_, reflection_->MutableMessage(message_, field, message_factory_))); - return absl::nullopt; + return std::nullopt; } case google::protobuf::Descriptor::WELLKNOWNTYPE_ANY: { // Probably not correct, need to use the parent/common one. @@ -1404,12 +1404,12 @@ class MessageValueBuilderImpl { well_known_types_.Any().SetValue( reflection_->MutableMessage(message_, field, message_factory_), std::move(serialized).Consume()); - return absl::nullopt; + return std::nullopt; } default: if (value.IsNull()) { // Allowing assigning `null` to message fields. - return absl::nullopt; + return std::nullopt; } break; } diff --git a/common/values/value_builder.cc b/common/values/value_builder.cc index 979837411..825fafeaf 100644 --- a/common/values/value_builder.cc +++ b/common/values/value_builder.cc @@ -1055,14 +1055,14 @@ class CompatMapValueImpl final : public CompatMapValue { CelValue key) const override { if (auto status = CelValue::CheckMapKeyType(key); !status.ok()) { status.IgnoreError(); - return absl::nullopt; + return std::nullopt; } if (auto it = map_.find(key); it != map_.end()) { return common_internal::UnsafeLegacyValue( it->second, /*stable=*/true, arena != nullptr ? arena : map_.get_allocator().arena()); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr Has(const CelValue& key) const override { @@ -1219,14 +1219,14 @@ class TrivialMutableMapValueImpl final : public MutableCompatMapValue { CelValue key) const override { if (auto status = CelValue::CheckMapKeyType(key); !status.ok()) { status.IgnoreError(); - return absl::nullopt; + return std::nullopt; } if (auto it = map_.find(key); it != map_.end()) { return common_internal::UnsafeLegacyValue( it->second, /*stable=*/false, arena != nullptr ? arena : map_.get_allocator().arena()); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr Has(const CelValue& key) const override { From aa564b2d2a73a6837f6dfe41d456fed845a1a3d6 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:08:43 -0700 Subject: [PATCH 102/350] No public description PiperOrigin-RevId: 942894299 --- eval/public/containers/container_backed_map_impl.cc | 2 +- eval/public/containers/internal_field_backed_map_impl.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/public/containers/container_backed_map_impl.cc b/eval/public/containers/container_backed_map_impl.cc index 5ac08af92..99d0cdeba 100644 --- a/eval/public/containers/container_backed_map_impl.cc +++ b/eval/public/containers/container_backed_map_impl.cc @@ -88,7 +88,7 @@ class CelValueEq { absl::optional CelMapBuilder::operator[](CelValue cel_key) const { auto item = values_map_.find(cel_key); if (item == values_map_.end()) { - return absl::nullopt; + return std::nullopt; } return item->second; } diff --git a/eval/public/containers/internal_field_backed_map_impl.cc b/eval/public/containers/internal_field_backed_map_impl.cc index a879955d1..2d5666c58 100644 --- a/eval/public/containers/internal_field_backed_map_impl.cc +++ b/eval/public/containers/internal_field_backed_map_impl.cc @@ -154,7 +154,7 @@ absl::optional FieldBackedMapImpl::operator[](CelValue key) const { return CreateErrorValue(arena_, lookup_result.status()); } if (!*lookup_result) { - return absl::nullopt; + return std::nullopt; } // Get value descriptor treating it as a repeated field. From 8056fd5f95ea316c3f448de1f3125f3dbc0ee222 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:08:51 -0700 Subject: [PATCH 103/350] No public description PiperOrigin-RevId: 942894346 --- runtime/standard/equality_functions.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime/standard/equality_functions.cc b/runtime/standard/equality_functions.cc index 6546db16c..4ff1acf9a 100644 --- a/runtime/standard/equality_functions.cc +++ b/runtime/standard/equality_functions.cc @@ -181,7 +181,7 @@ absl::optional NumberFromValue(const Value& value) { return Number::FromDouble(value.GetDouble().NativeValue()); } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> CheckAlternativeNumericType( @@ -192,7 +192,7 @@ absl::StatusOr> CheckAlternativeNumericType( absl::optional number = NumberFromValue(key); if (!number.has_value()) { - return absl::nullopt; + return std::nullopt; } if (!key.IsInt() && number->LosslessConvertibleToInt()) { @@ -215,7 +215,7 @@ absl::StatusOr> CheckAlternativeNumericType( } } - return absl::nullopt; + return std::nullopt; } // Equality for maps. Template parameter provides either heterogeneous or @@ -440,7 +440,7 @@ absl::StatusOr> HomogenousValueEqual( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { if (v1.kind() != v2.kind()) { - return absl::nullopt; + return std::nullopt; } static_assert(std::is_lvalue_reference_v, @@ -483,7 +483,7 @@ absl::StatusOr> HomogenousValueEqual( return OpaqueEqual(v1.GetOpaque(), v2.GetOpaque(), descriptor_pool, message_factory, arena); default: - return absl::nullopt; + return std::nullopt; } } @@ -582,7 +582,7 @@ absl::StatusOr> ValueEqualImpl( // map containing an Error. Return no matching overload to propagate an error // instead of a false result. if (v1.IsError() || v1.IsUnknown() || v2.IsError() || v2.IsUnknown()) { - return absl::nullopt; + return std::nullopt; } return false; From f697af360fea998393366d34986837f9d221c993 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:09:15 -0700 Subject: [PATCH 104/350] No public description PiperOrigin-RevId: 942894523 --- runtime/internal/runtime_type_provider.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/internal/runtime_type_provider.cc b/runtime/internal/runtime_type_provider.cc index 40f5ff575..e6fd55d2c 100644 --- a/runtime/internal/runtime_type_provider.cc +++ b/runtime/internal/runtime_type_provider.cc @@ -56,7 +56,7 @@ absl::StatusOr> RuntimeTypeProvider::FindTypeImpl( if (const auto it = types_.find(name); it != types_.end()) { return it->second; } - return absl::nullopt; + return std::nullopt; } absl::StatusOr> @@ -69,7 +69,7 @@ RuntimeTypeProvider::FindEnumConstantImpl(absl::string_view type, const google::protobuf::EnumDescriptor* enum_desc = descriptor_pool_->FindEnumTypeByName(type); if (enum_desc == nullptr) { - return absl::nullopt; + return std::nullopt; } // Note: we don't support strong enum typing at this time so only the fully @@ -78,7 +78,7 @@ RuntimeTypeProvider::FindEnumConstantImpl(absl::string_view type, const google::protobuf::EnumValueDescriptor* value_desc = enum_desc->FindValueByName(value); if (value_desc == nullptr) { - return absl::nullopt; + return std::nullopt; } return TypeIntrospector::EnumConstant{ @@ -95,13 +95,13 @@ RuntimeTypeProvider::FindStructTypeFieldByNameImpl( } const auto* desc = descriptor_pool_->FindMessageTypeByName(type); if (desc == nullptr) { - return absl::nullopt; + return std::nullopt; } const auto* field_desc = desc->FindFieldByName(name); if (field_desc == nullptr) { field_desc = descriptor_pool_->FindExtensionByPrintableName(desc, name); if (field_desc == nullptr) { - return absl::nullopt; + return std::nullopt; } } return MessageTypeField(field_desc); From 7ab0b81797238436ab2b730fcc82ec3ed51e2f54 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:09:25 -0700 Subject: [PATCH 105/350] No public description PiperOrigin-RevId: 942894574 --- eval/internal/cel_value_equal.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eval/internal/cel_value_equal.cc b/eval/internal/cel_value_equal.cc index f61f93ca4..b9f1c9cc7 100644 --- a/eval/internal/cel_value_equal.cc +++ b/eval/internal/cel_value_equal.cc @@ -101,7 +101,7 @@ absl::optional MapEqual(const CelMap* t1, const CelMap* t2) { google::protobuf::Arena arena; auto list_keys = t1->ListKeys(&arena); if (!list_keys.ok()) { - return absl::nullopt; + return std::nullopt; } const CelList* keys = *list_keys; for (int i = 0; i < keys->size(); i++) { @@ -166,7 +166,7 @@ template absl::optional HomogenousCelValueEqual(const CelValue& t1, const CelValue& t2) { if (t1.type() != t2.type()) { - return absl::nullopt; + return std::nullopt; } switch (t1.type()) { case Kind::kNullType: @@ -198,7 +198,7 @@ absl::optional HomogenousCelValueEqual(const CelValue& t1, default: break; } - return absl::nullopt; + return std::nullopt; } absl::optional HeterogeneousEqualProvider::operator()( @@ -233,7 +233,7 @@ absl::optional CelValueEqualImpl(const CelValue& v1, const CelValue& v2) { // map containing an Error. Return no matching overload to propagate an error // instead of a false result. if (v1.IsError() || v1.IsUnknownSet() || v2.IsError() || v2.IsUnknownSet()) { - return absl::nullopt; + return std::nullopt; } return false; From b5edead61047e38ffd69a8c66c17d78771a7f9dd Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:10:45 -0700 Subject: [PATCH 106/350] No public description PiperOrigin-RevId: 942894984 --- common/internal/byte_string.cc | 12 ++++++------ common/internal/byte_string_test.cc | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/common/internal/byte_string.cc b/common/internal/byte_string.cc index 304104a87..d35c0efa2 100644 --- a/common/internal/byte_string.cc +++ b/common/internal/byte_string.cc @@ -307,7 +307,7 @@ absl::optional ByteString::Find(absl::string_view needle, [&needle, pos](absl::string_view lhs) -> absl::optional { absl::string_view::size_type i = lhs.find(needle, pos); if (i == absl::string_view::npos) { - return absl::nullopt; + return std::nullopt; } return i; }, @@ -315,7 +315,7 @@ absl::optional ByteString::Find(absl::string_view needle, absl::Cord cord = lhs.Subcord(pos, lhs.size() - pos); absl::Cord::CharIterator it = cord.Find(needle); if (it == cord.char_end()) { - return absl::nullopt; + return std::nullopt; } return pos + static_cast(absl::Cord::Distance(cord.char_begin(), it)); @@ -331,14 +331,14 @@ absl::optional ByteString::Find(const absl::Cord& needle, if (auto flat_needle = needle.TryFlat(); flat_needle) { absl::string_view::size_type i = lhs.find(*flat_needle, pos); if (i == absl::string_view::npos) { - return absl::nullopt; + return std::nullopt; } return i; } // Needle is fragmented, we have to do a linear scan. const size_t needle_size = needle.size(); if (pos + needle_size > lhs.size()) { - return absl::nullopt; + return std::nullopt; } if (ABSL_PREDICT_FALSE(needle_size == 0)) { return pos; @@ -354,7 +354,7 @@ absl::optional ByteString::Find(const absl::Cord& needle, size_t found_pos = lhs.find(first_chunk, current_pos); if (found_pos == absl::string_view::npos || found_pos > lhs.size() - needle_size) { - return absl::nullopt; + return std::nullopt; } if (lhs.substr(found_pos + first_chunk.size(), rest_of_needle.size()) == rest_of_needle) { @@ -367,7 +367,7 @@ absl::optional ByteString::Find(const absl::Cord& needle, absl::Cord cord = lhs.Subcord(pos, lhs.size() - pos); absl::Cord::CharIterator it = cord.Find(needle); if (it == cord.char_end()) { - return absl::nullopt; + return std::nullopt; } return pos + static_cast(absl::Cord::Distance(cord.char_begin(), it)); diff --git a/common/internal/byte_string_test.cc b/common/internal/byte_string_test.cc index 553c9c13a..902a2e877 100644 --- a/common/internal/byte_string_test.cc +++ b/common/internal/byte_string_test.cc @@ -766,7 +766,7 @@ TEST_P(ByteStringTest, TryFlatLarge) { ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeFragmentedCord()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); - EXPECT_THAT(byte_string.TryFlat(), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.TryFlat(), Eq(std::nullopt)); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); } @@ -806,10 +806,10 @@ TEST_P(ByteStringTest, Find) { EXPECT_THAT( byte_string.Find("small string optimization!"), Optional(GetMediumStringView().find("small string optimization!"))); - EXPECT_THAT(byte_string.Find("not found"), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.Find("not found"), Eq(std::nullopt)); EXPECT_THAT(byte_string.Find(""), Optional(0)); EXPECT_THAT(byte_string.Find("", 3), Optional(3)); - EXPECT_THAT(byte_string.Find("A string", 1), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.Find("A string", 1), Eq(std::nullopt)); // Find cord EXPECT_THAT(byte_string.Find(absl::Cord("A string")), Optional(0)); @@ -820,25 +820,25 @@ TEST_P(ByteStringTest, Find) { byte_string.Find(absl::MakeFragmentedCord( {"A string", " that is too large for the small string optimization!", " extra"})), - Eq(absl::nullopt)); + Eq(std::nullopt)); EXPECT_THAT(byte_string.Find(GetMediumOrLargeFragmentedCord()), Optional(0)); - EXPECT_THAT(byte_string.Find(absl::Cord("not found")), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.Find(absl::Cord("not found")), Eq(std::nullopt)); EXPECT_THAT(byte_string.Find(absl::Cord("")), Optional(0)); EXPECT_THAT(byte_string.Find(absl::Cord(""), 3), Optional(3)); } TEST_P(ByteStringTest, FindEdgeCases) { ByteString empty_byte_string(GetAllocator(), ""); - EXPECT_THAT(empty_byte_string.Find("a"), Eq(absl::nullopt)); + EXPECT_THAT(empty_byte_string.Find("a"), Eq(std::nullopt)); EXPECT_THAT(empty_byte_string.Find(""), Optional(0)); ByteString cord_byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); - EXPECT_THAT(cord_byte_string.Find("not found"), Eq(absl::nullopt)); + EXPECT_THAT(cord_byte_string.Find("not found"), Eq(std::nullopt)); ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); // Needle longer than haystack. EXPECT_THAT(byte_string.Find(std::string(byte_string.size() + 1, 'a')), - Eq(absl::nullopt)); + Eq(std::nullopt)); // Needle at the end. absl::string_view suffix = "optimization!"; @@ -846,7 +846,7 @@ TEST_P(ByteStringTest, FindEdgeCases) { Optional(byte_string.size() - suffix.size())); // pos at the end. - EXPECT_THAT(byte_string.Find("a", byte_string.size()), Eq(absl::nullopt)); + EXPECT_THAT(byte_string.Find("a", byte_string.size()), Eq(std::nullopt)); EXPECT_THAT(byte_string.Find("", byte_string.size()), Optional(byte_string.size())); @@ -877,7 +877,7 @@ TEST_P(ByteStringTest, FindEdgeCases) { // enough space for the rest. ByteString short_haystack(GetAllocator(), "abcdefg"); absl::Cord needle_too_long = absl::MakeFragmentedCord({"ef", "gh"}); - EXPECT_THAT(short_haystack.Find(needle_too_long), Eq(absl::nullopt)); + EXPECT_THAT(short_haystack.Find(needle_too_long), Eq(std::nullopt)); // Search with a fragmented empty cord. absl::Cord fragmented_empty_cord = absl::MakeFragmentedCord({"", ""}); From b1fd0e673ceb502354b0fc0f8e9a3b22991a12c8 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:10:49 -0700 Subject: [PATCH 107/350] No public description PiperOrigin-RevId: 942895001 --- common/value.cc | 88 ++++++++++++++++++++++---------------------- common/value_test.cc | 44 +++++++++++----------- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/common/value.cc b/common/value.cc index 1cd3f54e1..f13ee958e 100644 --- a/common/value.cc +++ b/common/value.cc @@ -1941,14 +1941,14 @@ optional_ref Value::AsBytes() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsBytes() && { if (auto* alternative = variant_.As(); alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsDouble() const { @@ -1956,7 +1956,7 @@ absl::optional Value::AsDouble() const { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsDuration() const { @@ -1964,7 +1964,7 @@ absl::optional Value::AsDuration() const { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsError() const& { @@ -1972,14 +1972,14 @@ optional_ref Value::AsError() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsError() && { if (auto* alternative = variant_.As(); alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsInt() const { @@ -1987,7 +1987,7 @@ absl::optional Value::AsInt() const { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsList() const& { @@ -2007,7 +2007,7 @@ absl::optional Value::AsList() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsList() && { @@ -2027,7 +2027,7 @@ absl::optional Value::AsList() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsMap() const& { @@ -2047,7 +2047,7 @@ absl::optional Value::AsMap() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsMap() && { @@ -2067,7 +2067,7 @@ absl::optional Value::AsMap() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsMessage() const& { @@ -2075,7 +2075,7 @@ absl::optional Value::AsMessage() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsMessage() && { @@ -2083,7 +2083,7 @@ absl::optional Value::AsMessage() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsNull() const { @@ -2091,7 +2091,7 @@ absl::optional Value::AsNull() const { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsOpaque() const& { @@ -2099,14 +2099,14 @@ optional_ref Value::AsOpaque() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsOpaque() && { if (auto* alternative = variant_.As(); alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsOptional() const& { @@ -2114,7 +2114,7 @@ optional_ref Value::AsOptional() const& { alternative != nullptr && alternative->IsOptional()) { return static_cast(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsOptional() && { @@ -2122,7 +2122,7 @@ absl::optional Value::AsOptional() && { alternative != nullptr && alternative->IsOptional()) { return static_cast(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsParsedJsonList() const& { @@ -2130,7 +2130,7 @@ optional_ref Value::AsParsedJsonList() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsParsedJsonList() && { @@ -2138,7 +2138,7 @@ absl::optional Value::AsParsedJsonList() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsParsedJsonMap() const& { @@ -2146,7 +2146,7 @@ optional_ref Value::AsParsedJsonMap() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsParsedJsonMap() && { @@ -2154,7 +2154,7 @@ absl::optional Value::AsParsedJsonMap() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsCustomList() const& { @@ -2162,7 +2162,7 @@ optional_ref Value::AsCustomList() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsCustomList() && { @@ -2170,7 +2170,7 @@ absl::optional Value::AsCustomList() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsCustomMap() const& { @@ -2178,7 +2178,7 @@ optional_ref Value::AsCustomMap() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsCustomMap() && { @@ -2186,7 +2186,7 @@ absl::optional Value::AsCustomMap() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsParsedMapField() const& { @@ -2194,7 +2194,7 @@ optional_ref Value::AsParsedMapField() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsParsedMapField() && { @@ -2202,7 +2202,7 @@ absl::optional Value::AsParsedMapField() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsParsedMessage() const& { @@ -2210,7 +2210,7 @@ optional_ref Value::AsParsedMessage() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsParsedMessage() && { @@ -2218,7 +2218,7 @@ absl::optional Value::AsParsedMessage() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsParsedRepeatedField() @@ -2227,7 +2227,7 @@ optional_ref Value::AsParsedRepeatedField() alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsParsedRepeatedField() && { @@ -2235,7 +2235,7 @@ absl::optional Value::AsParsedRepeatedField() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsCustomStruct() const& { @@ -2243,7 +2243,7 @@ optional_ref Value::AsCustomStruct() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsCustomStruct() && { @@ -2251,7 +2251,7 @@ absl::optional Value::AsCustomStruct() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsString() const& { @@ -2259,14 +2259,14 @@ optional_ref Value::AsString() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsString() && { if (auto* alternative = variant_.As(); alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsStruct() const& { @@ -2283,7 +2283,7 @@ absl::optional Value::AsStruct() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsStruct() && { @@ -2299,7 +2299,7 @@ absl::optional Value::AsStruct() && { alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsTimestamp() const { @@ -2307,7 +2307,7 @@ absl::optional Value::AsTimestamp() const { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsType() const& { @@ -2315,14 +2315,14 @@ optional_ref Value::AsType() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsType() && { if (auto* alternative = variant_.As(); alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsUint() const { @@ -2330,7 +2330,7 @@ absl::optional Value::AsUint() const { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } optional_ref Value::AsUnknown() const& { @@ -2338,14 +2338,14 @@ optional_ref Value::AsUnknown() const& { alternative != nullptr) { return *alternative; } - return absl::nullopt; + return std::nullopt; } absl::optional Value::AsUnknown() && { if (auto* alternative = variant_.As(); alternative != nullptr) { return std::move(*alternative); } - return absl::nullopt; + return std::nullopt; } const BytesValue& Value::GetBytes() const& { diff --git a/common/value_test.cc b/common/value_test.cc index fb346423b..40ca590b0 100644 --- a/common/value_test.cc +++ b/common/value_test.cc @@ -190,7 +190,7 @@ TEST(Value, As) { google::protobuf::Arena arena; EXPECT_THAT(Value(BoolValue()).As(), Optional(An())); - EXPECT_THAT(Value(BoolValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(BoolValue()).As(), Eq(std::nullopt)); { Value value(BytesValue{}); @@ -207,11 +207,11 @@ TEST(Value, As) { EXPECT_THAT(Value(DoubleValue()).As(), Optional(An())); - EXPECT_THAT(Value(DoubleValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(DoubleValue()).As(), Eq(std::nullopt)); EXPECT_THAT(Value(DurationValue()).As(), Optional(An())); - EXPECT_THAT(Value(DurationValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(DurationValue()).As(), Eq(std::nullopt)); { Value value(ErrorValue{}); @@ -224,11 +224,11 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(ErrorValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(ErrorValue()).As(), Eq(std::nullopt)); } EXPECT_THAT(Value(IntValue()).As(), Optional(An())); - EXPECT_THAT(Value(IntValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(IntValue()).As(), Eq(std::nullopt)); { Value value(ListValue{}); @@ -241,7 +241,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(ListValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(ListValue()).As(), Eq(std::nullopt)); } { @@ -255,7 +255,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(ListValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(ListValue()).As(), Eq(std::nullopt)); } { @@ -282,7 +282,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(ListValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(ListValue()).As(), Eq(std::nullopt)); } { @@ -346,7 +346,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(MapValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(MapValue()).As(), Eq(std::nullopt)); } { @@ -360,7 +360,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(MapValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(MapValue()).As(), Eq(std::nullopt)); } { @@ -387,7 +387,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(MapValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(MapValue()).As(), Eq(std::nullopt)); } { @@ -460,11 +460,11 @@ TEST(Value, As) { GetTestingMessageFactory()), &arena}) .As(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } EXPECT_THAT(Value(NullValue()).As(), Optional(An())); - EXPECT_THAT(Value(NullValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(NullValue()).As(), Eq(std::nullopt)); { Value value(OptionalValue{}); @@ -478,7 +478,7 @@ TEST(Value, As) { EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); EXPECT_THAT(Value(OpaqueValue(OptionalValue())).As(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } { @@ -492,7 +492,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(OptionalValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(OptionalValue()).As(), Eq(std::nullopt)); } { @@ -536,7 +536,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(StringValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(StringValue()).As(), Eq(std::nullopt)); } { @@ -558,7 +558,7 @@ TEST(Value, As) { EXPECT_THAT(Value(TimestampValue()).As(), Optional(An())); - EXPECT_THAT(Value(TimestampValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(TimestampValue()).As(), Eq(std::nullopt)); { Value value(TypeValue(StringType{})); @@ -572,11 +572,11 @@ TEST(Value, As) { EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); EXPECT_THAT(Value(TypeValue(StringType())).As(), - Eq(absl::nullopt)); + Eq(std::nullopt)); } EXPECT_THAT(Value(UintValue()).As(), Optional(An())); - EXPECT_THAT(Value(UintValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(UintValue()).As(), Eq(std::nullopt)); { Value value(UnknownValue{}); @@ -589,7 +589,7 @@ TEST(Value, As) { Optional(An())); EXPECT_THAT(AsConstRValueRef(other_value).As(), Optional(An())); - EXPECT_THAT(Value(UnknownValue()).As(), Eq(absl::nullopt)); + EXPECT_THAT(Value(UnknownValue()).As(), Eq(std::nullopt)); } } @@ -985,13 +985,13 @@ TEST_F(ValueIteratorTest, Empty) { TEST_F(ValueIteratorTest, Empty1) { auto iterator = NewEmptyValueIterator(); EXPECT_THAT(iterator->Next1(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ValueIteratorTest, Empty2) { auto iterator = NewEmptyValueIterator(); EXPECT_THAT(iterator->Next2(descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } } // namespace From 2b84002039ed2adb37a5b69939e91d8736ec06a9 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 5 Jul 2026 12:10:58 -0700 Subject: [PATCH 108/350] No public description PiperOrigin-RevId: 942895058 --- runtime/activation.cc | 4 ++-- runtime/activation_test.cc | 14 +++++++------- runtime/embedder_context_test.cc | 8 ++++---- runtime/function_registry.cc | 2 +- runtime/function_registry_test.cc | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/runtime/activation.cc b/runtime/activation.cc index e999f7a02..2ce3eb364 100644 --- a/runtime/activation.cc +++ b/runtime/activation.cc @@ -102,14 +102,14 @@ std::vector Activation::FindFunctionOverloads( bool Activation::InsertOrAssignValue(absl::string_view name, Value value) { return values_ - .insert_or_assign(name, ValueEntry{std::move(value), absl::nullopt}) + .insert_or_assign(name, ValueEntry{std::move(value), std::nullopt}) .second; } bool Activation::InsertOrAssignValueProvider(absl::string_view name, ValueProvider provider) { return values_ - .insert_or_assign(name, ValueEntry{absl::nullopt, std::move(provider)}) + .insert_or_assign(name, ValueEntry{std::nullopt, std::move(provider)}) .second; } diff --git a/runtime/activation_test.cc b/runtime/activation_test.cc index 4303116a3..108ed7978 100644 --- a/runtime/activation_test.cc +++ b/runtime/activation_test.cc @@ -79,7 +79,7 @@ TEST_F(ActivationTest, ValueNotFound) { EXPECT_THAT(activation.FindVariable("var1", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ActivationTest, InsertValue) { @@ -122,11 +122,11 @@ TEST_F(ActivationTest, InsertProviderForwardsNotFound) { "var1", [](absl::string_view name, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - google::protobuf::Arena* absl_nonnull) { return absl::nullopt; })); + google::protobuf::Arena* absl_nonnull) { return std::nullopt; })); EXPECT_THAT(activation.FindVariable("var1", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); } TEST_F(ActivationTest, InsertProviderForwardsStatus) { @@ -355,10 +355,10 @@ TEST_F(ActivationTest, MoveAssignment) { // NOLINTBEGIN(bugprone-use-after-move) EXPECT_THAT(moved_from.FindVariable("val", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(moved_from.FindVariable("val_provided", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(moved_from.FindFunctionOverloads("Fn"), SizeIs(0)); EXPECT_THAT(moved_from.GetUnknownAttributes(), SizeIs(0)); EXPECT_THAT(moved_from.GetMissingAttributes(), SizeIs(0)); @@ -405,10 +405,10 @@ TEST_F(ActivationTest, MoveCtor) { // NOLINTBEGIN(bugprone-use-after-move) EXPECT_THAT(moved_from.FindVariable("val", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(moved_from.FindVariable("val_provided", descriptor_pool(), message_factory(), arena()), - IsOkAndHolds(Eq(absl::nullopt))); + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(moved_from.FindFunctionOverloads("Fn"), SizeIs(0)); EXPECT_THAT(moved_from.GetUnknownAttributes(), SizeIs(0)); EXPECT_THAT(moved_from.GetMissingAttributes(), SizeIs(0)); diff --git a/runtime/embedder_context_test.cc b/runtime/embedder_context_test.cc index d8cbbb736..4de80f5d3 100644 --- a/runtime/embedder_context_test.cc +++ b/runtime/embedder_context_test.cc @@ -29,11 +29,11 @@ TEST(EmbedderContextTest, From) { struct TestScope {}; EmbedderContext context = EmbedderContext::From(int64_t{42}); EXPECT_THAT((context.Get()), Optional(42)); - EXPECT_EQ((context.Get()), absl::nullopt); + EXPECT_EQ((context.Get()), std::nullopt); EmbedderContext context2 = EmbedderContext::From(uint64_t{42}); EXPECT_THAT((context2.Get()), Optional(42)); - EXPECT_EQ((context2.Get()), absl::nullopt); + EXPECT_EQ((context2.Get()), std::nullopt); // Side effect, but checking that we keep a dense range. EXPECT_EQ(cel::TypeIdInSet::Size(), 2); @@ -47,7 +47,7 @@ TEST(EmbedderContextTest, FromOutOfLine) { EXPECT_THAT((context.Get()), Optional(42)); EXPECT_THAT((context.Get()), Optional(43)); EXPECT_THAT((context.Get()), Optional(44)); - EXPECT_EQ((context.Get()), absl::nullopt); + EXPECT_EQ((context.Get()), std::nullopt); // Note: Referencing a type not intended to be stored will still reserve a // slot in the TypeIdInSet. @@ -76,7 +76,7 @@ TEST(EmbedderContextTest, FromPtrs) { TEST(EmbedderContextTest, FromDefaultScope) { EmbedderContext context = EmbedderContext::From(int64_t{42}); EXPECT_THAT((context.Get()), Optional(42)); - EXPECT_EQ((context.Get()), absl::nullopt); + EXPECT_EQ((context.Get()), std::nullopt); } // These death assertions are only enabled when compiled in debug mode. diff --git a/runtime/function_registry.cc b/runtime/function_registry.cc index 59f267255..b1f08e00a 100644 --- a/runtime/function_registry.cc +++ b/runtime/function_registry.cc @@ -50,7 +50,7 @@ class ActivationFunctionProviderImpl std::vector overloads = activation.FindFunctionOverloads(descriptor.name()); - std::optional matching_overload = absl::nullopt; + std::optional matching_overload = std::nullopt; for (const auto& overload : overloads) { if (overload.descriptor.ShapeMatches(descriptor)) { diff --git a/runtime/function_registry_test.cc b/runtime/function_registry_test.cc index 53916777a..40445f43a 100644 --- a/runtime/function_registry_test.cc +++ b/runtime/function_registry_test.cc @@ -124,7 +124,7 @@ TEST(FunctionRegistryTest, DefaultLazyProviderNoOverloadFound) { provider.GetFunction({"LazyFunc", false, {cel::Kind::kInt64}}, activation)); - EXPECT_EQ(func, absl::nullopt); + EXPECT_EQ(func, std::nullopt); } TEST(FunctionRegistryTest, DefaultLazyProviderReturnsImpl) { From 2e2be52ae3eaa3b32e5743e0c633e7e83ed0164d Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 6 Jul 2026 09:40:27 -0700 Subject: [PATCH 109/350] Add basic macos bazel test workflow Runs on post merge to main branch. Closes #2117 PiperOrigin-RevId: 943339994 --- .github/workflows/bazel_test_post_merge.yml | 16 +++++++++++ .github/workflows/macos_bazel_test.yml | 27 +++++++++++++++++++ .github/workflows/windows_bazel_test.yml | 2 +- .../windows_bazel_test_post_merge.yml | 13 --------- 4 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/bazel_test_post_merge.yml create mode 100644 .github/workflows/macos_bazel_test.yml delete mode 100644 .github/workflows/windows_bazel_test_post_merge.yml diff --git a/.github/workflows/bazel_test_post_merge.yml b/.github/workflows/bazel_test_post_merge.yml new file mode 100644 index 000000000..d9cd1b294 --- /dev/null +++ b/.github/workflows/bazel_test_post_merge.yml @@ -0,0 +1,16 @@ +name: Bazel Test (Post-Merge) + +on: + push: + branches: + - master + +jobs: + trigger-win-latest-test: + # This prevents the workflow from running automatically when someone + # pushes to their fork. + if: github.repository == 'cel-expr/cel-cpp' + uses: ./.github/workflows/windows_bazel_test.yml + trigger-mac-latest-test: + if: github.repository == 'cel-expr/cel-cpp' + uses: ./.github/workflows/macos_bazel_test.yml diff --git a/.github/workflows/macos_bazel_test.yml b/.github/workflows/macos_bazel_test.yml new file mode 100644 index 000000000..24763f53a --- /dev/null +++ b/.github/workflows/macos_bazel_test.yml @@ -0,0 +1,27 @@ +name: MacOS Bazel Test + +on: + workflow_call: + workflow_dispatch: + +jobs: + test: + name: Run Bazel Tests + runs-on: macos-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Bazel and Bazelisk + uses: bazel-contrib/setup-bazel@0.19.0 + with: + bazelisk-cache: true + disk-cache: ${{ github.workflow }} + repository-cache: true + + - name: Run Tests + shell: bash + # TODO(b/506179116): is not supported on all target versions in xcode, so not + # running policy conformance. + run: | + bazelisk test --copt=-Wno-deprecated-declarations --copt=-Wno-nullability-completeness conformance:all diff --git a/.github/workflows/windows_bazel_test.yml b/.github/workflows/windows_bazel_test.yml index 6d12e6861..4fb93c04a 100644 --- a/.github/workflows/windows_bazel_test.yml +++ b/.github/workflows/windows_bazel_test.yml @@ -25,4 +25,4 @@ jobs: # //... won't work. shell: bash run: | - bazelisk test --config=msvc conformance:all conformance/policy:all \ No newline at end of file + bazelisk test --config=msvc conformance:all conformance/policy:all diff --git a/.github/workflows/windows_bazel_test_post_merge.yml b/.github/workflows/windows_bazel_test_post_merge.yml deleted file mode 100644 index 569177fcc..000000000 --- a/.github/workflows/windows_bazel_test_post_merge.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Windows Bazel Test (Post-Merge) - -on: - push: - branches: - - master - -jobs: - trigger-test: - # This prevents the workflow from running automatically when someone - # pushes to their fork. - if: github.repository == 'cel-expr/cel-cpp' - uses: ./.github/workflows/windows_bazel_test.yml \ No newline at end of file From a1721f330ae3f83dec05d6974c2f8dd8f6952f50 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 6 Jul 2026 11:24:40 -0700 Subject: [PATCH 110/350] Re-export cloud build docker file PiperOrigin-RevId: 943399362 --- ci/ubuntu_floor/Dockerfile | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 ci/ubuntu_floor/Dockerfile diff --git a/ci/ubuntu_floor/Dockerfile b/ci/ubuntu_floor/Dockerfile new file mode 100644 index 000000000..dd81bd45e --- /dev/null +++ b/ci/ubuntu_floor/Dockerfile @@ -0,0 +1,70 @@ +# This Dockerfile is used to create a container with compilers and tools +# building the CEL C++ library on GitHub. +# +# To update a new version of this container, use gcloud. You may need to run +# `gcloud auth login` and `gcloud auth configure-docker` first. +# +# Note, if you need to run docker using `sudo` use the following commands +# instead: +# +# sudo gcloud auth login --no-launch-browser +# sudo gcloud auth configure-docker +# +# Run the following command from the root of the CEL repository: +# +# gcloud builds submit --region=us -t gcr.io/cel-analysis/cel-cpp/ubuntu_floor . +# +# Once complete get the sha256 digest from the output using the following +# command: +# +# gcloud artifacts versions list --package=cel-cpp/ubuntu_floor --repository=gcr.io \ +# --location=us +# +# The cloudbuild.yaml file must be updated to use the new digest like so: +# +# - name: 'gcr.io/cel-analysis/cel-cpp/ubuntu_floor@' +FROM gcr.io/cloud-marketplace/google/ubuntu2204:latest + +# Install Bazel prerequesites and required tools. +# See https://docs.bazel.build/versions/master/install-ubuntu.html +# See https://github.com/google/oss-policies-info/blob/main/foundational-cxx-support-matrix.md +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + git \ + libssl-dev \ + make \ + pkg-config \ + python3 \ + unzip \ + wget \ + zip \ + zlib1g-dev \ + default-jdk-headless \ + clang-14 \ + gcc-10 g++-10 \ + tzdata \ + && apt-get clean + +# Install Bazelisk. +# https://github.com/bazelbuild/bazelisk/releases +ARG BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/v1.27.0/bazelisk-amd64.deb" +ARG BAZELISK_CHKSUM="d8b00ea975c823e15263c80200ac42979e17368547fbff4ab177af035badfa83" +ADD ${BAZELISK_URL} /tmp/bazelisk.deb + +ENV BAZELISK_CHKSUM=${BAZELISK_CHKSUM} +RUN echo "${BAZELISK_CHKSUM} */tmp/bazelisk.deb" | sha256sum --check + +RUN apt-get install /tmp/bazelisk.deb + +RUN mkdir -p /workspace +RUN mkdir -p /bazel + +RUN USE_BAZEL_VERSION=8.7.0 bazelisk help +RUN USE_BAZEL_VERSION=7.3.2 bazelisk help + +ENV CC=gcc-10 +ENV CXX=g++-10 + +ENTRYPOINT ["/usr/bin/bazelisk"] From 469628f0a9d8f2fc9f3d49286c452b573a782437 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 7 Jul 2026 15:48:40 -0700 Subject: [PATCH 111/350] Update policy conformance tests to use the bundled format. PiperOrigin-RevId: 944135689 --- MODULE.bazel | 4 +- conformance/policy/BUILD | 32 +++- .../policy/policy_conformance_test.bzl | 9 +- conformance/policy/policy_conformance_test.cc | 177 ++++++++++-------- 4 files changed, 132 insertions(+), 90 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 703063c8d..970b719bd 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -98,9 +98,9 @@ bazel_dep( version = "0.9.0", ) -_CEL_POLICY_TAG = "ebfb2361f47080af643c14cf4da4c2b551a68740" +_CEL_POLICY_TAG = "4c2c2fd5b8f20da4eb34e3ee82a4ef1b7f023347" -_CEL_POLICY_SHA = "ea69e9c6b7bd5bc37d358148aebd2fcca38bc7c45a23feb635de72338e0327c1" +_CEL_POLICY_SHA = "e2bd943f89a3cab3b6508daf0bc5aa53d00e3e6ea52ef14a1d6f618a2b2909be" http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") diff --git a/conformance/policy/BUILD b/conformance/policy/BUILD index 29210e02d..e5d503d88 100644 --- a/conformance/policy/BUILD +++ b/conformance/policy/BUILD @@ -64,15 +64,39 @@ cc_library( alwayslink = True, ) +_ALL_TESTS = [ + "@cel_policy//conformance:k8s", + "@cel_policy//conformance:limits", + "@cel_policy//conformance:nested_rule", + "@cel_policy//conformance:nested_rule2", + "@cel_policy//conformance:nested_rule3", + "@cel_policy//conformance:nested_rule4", + "@cel_policy//conformance:nested_rule5", + "@cel_policy//conformance:nested_rule6", + "@cel_policy//conformance:nested_rule7", + "@cel_policy//conformance:nested_rules_variable_shadowing", + "@cel_policy//conformance:pb", + "@cel_policy//conformance:required_labels", + "@cel_policy//conformance:restricted_destinations", + "@cel_policy//conformance:unconditional_rules", + "@cel_policy//conformance:unnest", + "@cel_policy//conformance:variable_type_propagation", + "@cel_policy//conformance:compile_error_compose_conflicting_output", + "@cel_policy//conformance:compile_error_compose_conflicting_subrule", + "@cel_policy//conformance:compile_error_unreachable", + "@cel_policy//conformance:compile_error_duplicate_variable", + "@cel_policy//conformance:compile_error_import", + "@cel_policy//conformance:compile_error_incompatible_outputs", + "@cel_policy//conformance:compile_error_syntax", + "@cel_policy//conformance:compile_error_undeclared_reference", +] + cel_policy_conformance_test( name = "policy_conformance_test", - example = "@cel_policy//conformance:testdata/nested_rule/policy.yaml", skip_tests = [ # TODO(b/506179116): Fix these. # Need to add k8s custom yaml parser and mock runtime. "k8s", ], - test_files = [ - "@cel_policy//conformance:testdata", - ], + test_files = _ALL_TESTS, ) diff --git a/conformance/policy/policy_conformance_test.bzl b/conformance/policy/policy_conformance_test.bzl index 0b4d1a4c6..e57ac7be4 100644 --- a/conformance/policy/policy_conformance_test.bzl +++ b/conformance/policy/policy_conformance_test.bzl @@ -18,26 +18,25 @@ This module contains build rules for generating policy conformance test targets. load("@rules_cc//cc:cc_test.bzl", "cc_test") -def cel_policy_conformance_test(name, test_files, example, skip_tests = [], **kwargs): +def cel_policy_conformance_test(name, test_files, skip_tests = [], **kwargs): """Generates a policy conformance test target. Args: name: Name of the test target. test_files: List of targets or files representing the test data. - example: A specific example file from test_files used for runfiles resolution. skip_tests: List of test cases to skip. - testdata_dir: Path to testdata directory under runfiles. **kwargs: Additional arguments passed to the underlying cc_test. """ args = ["--gunit_fail_if_no_test_linked"] - args.append("--testdata_example='$(rlocationpath {})'".format(example)) + paths = ["$(rlocationpath {})".format(f) for f in test_files] + args.append("--test_bundles=\"{}\"".format(",".join(paths))) if skip_tests: args.append("--skip_tests=" + ",".join(skip_tests)) cc_test( name = name, - data = test_files + [example], + data = test_files, deps = [ "//conformance/policy:policy_conformance_test_lib", ], diff --git a/conformance/policy/policy_conformance_test.cc b/conformance/policy/policy_conformance_test.cc index 0d68f8abf..ef53ab34e 100644 --- a/conformance/policy/policy_conformance_test.cc +++ b/conformance/policy/policy_conformance_test.cc @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include #include #include -// NOLINTNEXTLINE(build/c++17) for OSS compatibility -#include #include "cel/expr/eval.pb.h" #include "absl/flags/flag.h" @@ -28,8 +27,8 @@ #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" +#include "absl/strings/strip.h" #include "common/ast.h" #include "common/internal/value_conversion.h" #include "common/source.h" @@ -61,12 +60,9 @@ #include "google/protobuf/descriptor.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" -#include "google/protobuf/text_format.h" -// Use a specific file to handle bazel runfiles resolution correctly. We find -// parent directory named 'testdata' to use as the root of the test cases. -ABSL_FLAG(std::string, testdata_example, "", - "Path to a specific example file."); +ABSL_FLAG(std::vector, test_bundles, {}, + "Space or comma separated list of test bundle runfiles paths."); ABSL_FLAG(std::vector, skip_tests, {}, "Comma-separated list of tests to skip."); @@ -78,6 +74,63 @@ using ::cel::expr::conformance::test::TestSuite; using ::cel::internal::GetSharedTestingDescriptorPool; using ::testing::HasSubstr; +struct BundleSections { + absl::string_view config_content; + absl::string_view policy_content; + absl::string_view tests_content; +}; + +absl::string_view TrimDoc(absl::string_view doc) { + absl::ConsumeSuffix(&doc, "\n"); + absl::ConsumeSuffix(&doc, "\r"); + return doc; +} + +absl::StatusOr ParseYamlBundle( + absl::string_view bundle_content) { + BundleSections sections; + std::vector docs; + absl::string_view remaining = bundle_content; + + size_t next_line = remaining.find('\n'); + while (next_line != absl::string_view::npos) { + if (absl::StartsWith(remaining.substr(next_line), "\n---\r\n")) { + docs.push_back(TrimDoc(remaining.substr(0, next_line))); + remaining = remaining.substr(next_line + 5); + next_line = remaining.find('\n'); + continue; + } + + if (absl::StartsWith(remaining.substr(next_line), "\n---\n")) { + docs.push_back(TrimDoc(remaining.substr(0, next_line))); + remaining = remaining.substr(next_line + 4); + next_line = remaining.find('\n'); + continue; + } + + next_line = remaining.find('\n', next_line + 1); + } + + if (remaining.empty()) { + return absl::InvalidArgumentError("Empty document in yaml bundle"); + } + docs.push_back(remaining); + + if (docs.size() == 3) { + sections.config_content = docs[0]; + sections.policy_content = docs[1]; + sections.tests_content = docs[2]; + } else if (docs.size() == 2) { + sections.policy_content = docs[0]; + sections.tests_content = docs[1]; + } else { + return absl::InvalidArgumentError( + absl::StrCat("Invalid number of sections: ", docs.size())); + } + + return sections; +} + // Implementations for extension functions referenced in conformance tests. cel::Value LocationCode(const cel::StringValue& ip, const google::protobuf::DescriptorPool* pool, @@ -447,9 +500,8 @@ class CelPolicyTest : public testing::Test { bool skip_; }; - absl::Status RegisterTestSuite( - const std::filesystem::path& dir_path, const std::string& suite_name, + absl::string_view suite_name, const BundleSections& sections, const std::shared_ptr& input_evaluator, const std::shared_ptr& pool, const std::shared_ptr& message_factory, @@ -463,28 +515,15 @@ absl::Status RegisterTestSuite( } } - std::filesystem::path policy_path = dir_path / "policy.yaml"; - std::filesystem::path tests_path = dir_path / "tests.yaml"; - bool is_yaml = true; - if (!std::filesystem::exists(tests_path)) { - tests_path = dir_path / "tests.textproto"; - is_yaml = false; - } - std::filesystem::path config_path = dir_path / "config.yaml"; - - if (!std::filesystem::exists(policy_path) || - !std::filesystem::exists(tests_path)) { - // Not a valid test suite, assume it's a directory we don't care about. + if (sections.policy_content.empty() || sections.tests_content.empty()) { return absl::OkStatus(); } // Parse Environment Config cel::Config config; - if (std::filesystem::exists(config_path)) { - std::string config_content; - CEL_RETURN_IF_ERROR( - cel::internal::GetFileContents(config_path.string(), &config_content)); - CEL_ASSIGN_OR_RETURN(config, cel::EnvConfigFromYaml(config_content)); + if (!sections.config_content.empty()) { + CEL_ASSIGN_OR_RETURN( + config, cel::EnvConfigFromYaml(std::string(sections.config_content))); } // Enable default extensions (optional, bindings) in the config @@ -537,11 +576,8 @@ absl::Status RegisterTestSuite( CEL_ASSIGN_OR_RETURN(auto runtime, std::move(runtime_builder).Build()); // Parse Policy - std::string policy_content; - CEL_RETURN_IF_ERROR( - cel::internal::GetFileContents(policy_path.string(), &policy_content)); CEL_ASSIGN_OR_RETURN(auto source, - cel::NewSource(policy_content, "policy.yaml")); + cel::NewSource(sections.policy_content, "policy.yaml")); auto policy_source = std::make_shared(std::move(source)); CEL_ASSIGN_OR_RETURN(CelPolicyParseResult parse_result, cel::ParseYamlCelPolicy(policy_source)); @@ -556,25 +592,16 @@ absl::Status RegisterTestSuite( CEL_ASSIGN_OR_RETURN(CelPolicyValidationResult compile_result, CompilePolicy(*compiler, *policy)); - std::string tests_content; - CEL_RETURN_IF_ERROR( - cel::internal::GetFileContents(tests_path.string(), &tests_content)); TestSuite test_suite; - if (is_yaml) { - CEL_ASSIGN_OR_RETURN(test_suite, - cel::test::ParsePolicyTestSuiteYaml(tests_content)); - } else { - if (!google::protobuf::TextFormat::ParseFromString(tests_content, &test_suite)) { - return absl::InvalidArgumentError( - absl::StrCat("Failed to parse text proto in ", tests_path.string())); - } - } + CEL_ASSIGN_OR_RETURN( + test_suite, cel::test::ParsePolicyTestSuiteYaml(sections.tests_content)); + std::string suite_name_str(suite_name); auto runner = std::make_shared( - suite_name, std::move(compiler), std::move(runtime), + suite_name_str, std::move(compiler), std::move(runtime), std::move(policy_source), std::move(compile_result), pool, message_factory, input_evaluator, config.GetContextType(), - /*expect_compile_fail=*/absl::StrContains(suite_name, "compile_errors")); + /*expect_compile_fail=*/absl::StrContains(suite_name, "compile_error")); for (const auto& section : test_suite.sections()) { std::string section_name = section.name(); @@ -586,7 +613,7 @@ absl::Status RegisterTestSuite( bool skip = !ShouldRunTest(full_test_name, skip_tests); testing::RegisterTest( - suite_name.c_str(), + suite_name_str.c_str(), absl::StrCat(section_name, "/", test_name).c_str(), nullptr, test_name.c_str(), __FILE__, __LINE__, [runner, test, full_test_name, skip]() -> CelPolicyTest* { @@ -598,14 +625,11 @@ absl::Status RegisterTestSuite( } void RegisterAllTests() { - // cel::google3-end - std::string testdata_example_flag = absl::GetFlag(FLAGS_testdata_example); + std::vector bundle_paths = absl::GetFlag(FLAGS_test_bundles); std::vector skip_tests = absl::GetFlag(FLAGS_skip_tests); - std::string abs_testdata_example = - cel::internal::ResolveRunfilesPath(testdata_example_flag); - ABSL_CHECK(!abs_testdata_example.empty()) - << "Could not find testdata directory: " << testdata_example_flag; + ABSL_CHECK(!bundle_paths.empty()) + << "No test bundles specified in --test_bundles flag."; std::shared_ptr pool = GetSharedTestingDescriptorPool(); @@ -616,35 +640,30 @@ void RegisterAllTests() { ABSL_CHECK_OK(evaluator_or.status()) << "Failed to create input evaluator"; std::shared_ptr evaluator = std::move(evaluator_or.value()); - std::filesystem::path testdata_path(abs_testdata_example); - ABSL_CHECK(std::filesystem::exists(testdata_path)) - << "Testdata path does not exist: " << testdata_path; - // walk up to find 'testdata' parent. A work around to portably - // get the expected directory from bazel. - while (!absl::EndsWith(testdata_path.string(), "testdata")) { - testdata_path = testdata_path.parent_path(); - ABSL_CHECK(testdata_path.string().size() > sizeof("testdata")) - << "could not resolve testdata directory"; - } + for (const std::string& bundle_path : bundle_paths) { + std::string abs_path = cel::internal::ResolveRunfilesPath(bundle_path); + ABSL_CHECK(!abs_path.empty()) + << "Could not resolve runfile path for test bundle: " << bundle_path; - for (const auto& entry : - std::filesystem::recursive_directory_iterator(testdata_path)) { - if (!entry.is_directory()) { - continue; - } - std::filesystem::path dir_path = entry.path(); - // Check if this directory has policy.yaml and tests.yaml (or - // tests.textproto) - if (std::filesystem::exists(dir_path / "policy.yaml") && - (std::filesystem::exists(dir_path / "tests.yaml") || - std::filesystem::exists(dir_path / "tests.textproto"))) { - std::string suite_name = absl::StrReplaceAll( - std::filesystem::relative(dir_path, testdata_path).string(), - {{"\\", "/"}}); - - ABSL_CHECK_OK(RegisterTestSuite(dir_path, suite_name, evaluator, pool, - message_factory, skip_tests)); + std::string bundle_content; + ABSL_CHECK_OK(cel::internal::GetFileContents(abs_path, &bundle_content)) + << "Failed to read bundle file: " << abs_path; + + auto sections_or = ParseYamlBundle(bundle_content); + ABSL_CHECK_OK(sections_or.status()) + << "Failed to parse bundle file: " << abs_path; + + absl::string_view filename = bundle_path; + size_t last_slash = filename.find_last_of("/\\"); + if (last_slash != absl::string_view::npos) { + filename = filename.substr(last_slash + 1); } + absl::string_view suite_view = filename; + absl::ConsumeSuffix(&suite_view, "_bundle.yaml"); + std::string suite_name = std::string(suite_view); + + ABSL_CHECK_OK(RegisterTestSuite(suite_name, sections_or.value(), evaluator, + pool, message_factory, skip_tests)); } } From 753fb7e6f6c7ceb642f6d065afad6a4243d156b7 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 8 Jul 2026 11:09:06 -0700 Subject: [PATCH 112/350] Add policy conformance test to macos post submit Bump bazel deps. PiperOrigin-RevId: 944596659 --- .github/workflows/macos_bazel_test.yml | 2 +- MODULE.bazel | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos_bazel_test.yml b/.github/workflows/macos_bazel_test.yml index 24763f53a..c0d448503 100644 --- a/.github/workflows/macos_bazel_test.yml +++ b/.github/workflows/macos_bazel_test.yml @@ -24,4 +24,4 @@ jobs: # TODO(b/506179116): is not supported on all target versions in xcode, so not # running policy conformance. run: | - bazelisk test --copt=-Wno-deprecated-declarations --copt=-Wno-nullability-completeness conformance:all + bazelisk test --copt=-Wno-deprecated-declarations --copt=-Wno-nullability-completeness conformance:all conformance/policy:all diff --git a/MODULE.bazel b/MODULE.bazel index 970b719bd..0901db5f8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -17,11 +17,11 @@ bazel_dep( ) bazel_dep( name = "rules_cc", - version = "0.2.14", + version = "0.2.18", ) bazel_dep( name = "rules_java", - version = "8.6.1", + version = "8.14.0", ) bazel_dep( name = "rules_proto", @@ -69,7 +69,7 @@ bazel_dep( ) bazel_dep( name = "platforms", - version = "1.0.0", + version = "1.1.0", ) bazel_dep( name = "antlr4-cpp-runtime", From 610847962ae24b034e094179e4cd08a6f9e0aef0 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 9 Jul 2026 11:58:50 -0700 Subject: [PATCH 113/350] Generalize cleanup and jump steps for and/or. PiperOrigin-RevId: 945241913 --- eval/compiler/BUILD | 4 +- eval/compiler/flat_expr_builder.cc | 6 +- ...ilder_short_circuiting_conformance_test.cc | 134 +++++------------- eval/eval/jump_step.cc | 62 +++++--- eval/eval/jump_step.h | 15 +- eval/eval/logic_step.cc | 66 +++++---- 6 files changed, 130 insertions(+), 157 deletions(-) diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index f7300cb58..ea4dd46b3 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -501,7 +501,6 @@ cc_test( ], deps = [ ":cel_expression_builder_flat_impl", - "//base:builtins", "//eval/public:activation", "//eval/public:cel_attribute", "//eval/public:cel_expression", @@ -509,8 +508,11 @@ cc_test( "//eval/public:unknown_attribute_set", "//eval/public:unknown_set", "//internal:testing", + "//parser", + "//parser:options", "//runtime:runtime_options", "//runtime/internal:runtime_env_testing", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index aa9a8858c..9cb9fbb01 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -2193,10 +2193,10 @@ void BinaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { std::unique_ptr jump_step; switch (cond_) { case BinaryCond::kAnd: - jump_step = CreateCondJumpStep(false, true, {}, expr->id()); + jump_step = CreateCondJumpStep(false, {}, expr->id()); break; case BinaryCond::kOr: - jump_step = CreateCondJumpStep(true, true, {}, expr->id()); + jump_step = CreateCondJumpStep(true, {}, expr->id()); break; default: ABSL_UNREACHABLE(); @@ -2321,7 +2321,7 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { // Value is to be removed from the stack. ProgramStepIndex cond_jump_pos = visitor_->GetCurrentIndex(); auto* jump_to_second = - visitor_->AddStep(CreateCondJumpStep(false, false, {}, expr->id())); + visitor_->AddStep(CreateTernaryCondJumpStep({}, expr->id())); if (jump_to_second) { jump_to_second_ = Jump(cond_jump_pos, static_cast(jump_to_second)); diff --git a/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc b/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc index afe7c5f9f..c00b66824 100644 --- a/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc +++ b/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc @@ -1,11 +1,13 @@ // A collection of tests that confirm that short-circuit and non-short-circuit // produce expressions with the same outputs. #include +#include +#include +#include "absl/log/absl_check.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "base/builtins.h" #include "eval/compiler/cel_expression_builder_flat_impl.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" @@ -14,10 +16,11 @@ #include "eval/public/unknown_attribute_set.h" #include "eval/public/unknown_set.h" #include "internal/testing.h" +#include "parser/options.h" +#include "parser/parser.h" #include "runtime/internal/runtime_env_testing.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" -#include "google/protobuf/text_format.h" namespace google::api::expr::runtime { @@ -25,64 +28,10 @@ namespace { using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::Expr; +using ::google::api::expr::parser::Parse; using ::testing::Eq; using ::testing::SizeIs; -constexpr char kTwoLogicalOp[] = R"cel( -id: 1 -call_expr { - function: "$0" - args { - id: 2 - ident_expr { - name: "var1", - } - } - args { - id: 3 - call_expr { - function: "$0" - args { - id: 4 - ident_expr { - name: "var2" - } - } - args { - id: 5 - ident_expr { - name: "var3" - } - } - } - } -} -)cel"; - -constexpr char kTernaryExpr[] = R"cel( -id: 1 -call_expr { - function: "_?_:_" - args { - id: 2 - ident_expr { - name: "cond" - } - } - args { - id: 3 - ident_expr { - name: "arg1" - } - } - args { - id: 4 - ident_expr { - name: "arg2" - } - } -})cel"; - void BuildAndEval(CelExpressionBuilder* builder, const Expr& expr, const Activation& activation, google::protobuf::Arena* arena, CelValue* result) { @@ -95,12 +44,16 @@ void BuildAndEval(CelExpressionBuilder* builder, const Expr& expr, *result = *value; } -class ShortCircuitingTest : public testing::TestWithParam { +class ShortCircuitingTest + : public testing::TestWithParam> { public: + bool short_circuiting() const { return std::get<0>(GetParam()); } + bool enable_variadic() const { return std::get<1>(GetParam()); } + std::unique_ptr GetBuilder( bool enable_unknowns = false) { cel::RuntimeOptions options; - options.short_circuiting = GetParam(); + options.short_circuiting = short_circuiting(); if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeAndFunction; @@ -109,14 +62,20 @@ class ShortCircuitingTest : public testing::TestWithParam { NewTestingRuntimeEnv(), options); return result; } + + Expr ParseExpr(absl::string_view expression) { + cel::ParserOptions options; + options.enable_variadic_logical_operators = enable_variadic(); + auto parsed_expr = Parse(expression, "", options); + ABSL_CHECK_OK(parsed_expr.status()); + return parsed_expr->expr(); + } }; TEST_P(ShortCircuitingTest, BasicAnd) { - Expr expr; + Expr expr = ParseExpr("var1 && var2 && var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kAnd), &expr)); auto builder = GetBuilder(); activation.InsertValue("var1", CelValue::CreateBool(true)); @@ -140,11 +99,9 @@ TEST_P(ShortCircuitingTest, BasicAnd) { } TEST_P(ShortCircuitingTest, BasicOr) { - Expr expr; + Expr expr = ParseExpr("var1 || var2 || var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kOr), &expr)); auto builder = GetBuilder(); activation.InsertValue("var1", CelValue::CreateBool(false)); @@ -168,11 +125,9 @@ TEST_P(ShortCircuitingTest, BasicOr) { } TEST_P(ShortCircuitingTest, ErrorAnd) { - Expr expr; + Expr expr = ParseExpr("var1 && var2 && var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kAnd), &expr)); auto builder = GetBuilder(); absl::Status error = absl::InternalError("error"); @@ -198,11 +153,9 @@ TEST_P(ShortCircuitingTest, ErrorAnd) { } TEST_P(ShortCircuitingTest, ErrorOr) { - Expr expr; + Expr expr = ParseExpr("var1 || var2 || var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kOr), &expr)); auto builder = GetBuilder(); absl::Status error = absl::InternalError("error"); @@ -228,11 +181,9 @@ TEST_P(ShortCircuitingTest, ErrorOr) { } TEST_P(ShortCircuitingTest, UnknownAnd) { - Expr expr; + Expr expr = ParseExpr("var1 && var2 && var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kAnd), &expr)); auto builder = GetBuilder(/* enable_unknowns=*/true); absl::Status error = absl::InternalError("error"); @@ -260,11 +211,9 @@ TEST_P(ShortCircuitingTest, UnknownAnd) { } TEST_P(ShortCircuitingTest, UnknownOr) { - Expr expr; + Expr expr = ParseExpr("var1 || var2 || var3"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - absl::Substitute(kTwoLogicalOp, ::cel::builtin::kOr), &expr)); auto builder = GetBuilder(/* enable_unknowns=*/true); absl::Status error = absl::InternalError("error"); @@ -292,10 +241,9 @@ TEST_P(ShortCircuitingTest, UnknownOr) { } TEST_P(ShortCircuitingTest, BasicTernary) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(); activation.InsertValue("cond", CelValue::CreateBool(true)); @@ -319,10 +267,9 @@ TEST_P(ShortCircuitingTest, BasicTernary) { } TEST_P(ShortCircuitingTest, TernaryErrorHandling) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(); absl::Status error1 = absl::InternalError("error1"); @@ -349,10 +296,9 @@ TEST_P(ShortCircuitingTest, TernaryErrorHandling) { } TEST_P(ShortCircuitingTest, TernaryUnknownCondHandling) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(/*enable_unknowns=*/true); absl::Status error = absl::InternalError("error1"); @@ -386,10 +332,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownCondHandling) { } TEST_P(ShortCircuitingTest, TernaryUnknownArgsHandling) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(/*enable_unknowns=*/true); absl::Status error = absl::InternalError("error1"); @@ -421,10 +366,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownArgsHandling) { } TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { - Expr expr; + Expr expr = ParseExpr("cond ? arg1 : arg2"); Activation activation; google::protobuf::Arena arena; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kTernaryExpr, &expr)); auto builder = GetBuilder(/*enable_unknowns=*/true); absl::Status error = absl::InternalError("error1"); @@ -457,16 +401,16 @@ TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { EXPECT_EQ(attrs.begin()->variable_name(), "cond"); } -const char* TestName(testing::TestParamInfo info) { - if (info.param) { - return "short_circuit_enabled"; - } else { - return "short_circuit_disabled"; - } +std::string TestName(testing::TestParamInfo> info) { + return absl::StrCat( + std::get<0>(info.param) ? "short_circuit_enabled" + : "short_circuit_disabled", + "_", std::get<1>(info.param) ? "variadic_enabled" : "variadic_disabled"); } INSTANTIATE_TEST_SUITE_P(Test, ShortCircuitingTest, - testing::Values(false, true), &TestName); + testing::Combine(testing::Bool(), testing::Bool()), + &TestName); } // namespace diff --git a/eval/eval/jump_step.cc b/eval/eval/jump_step.cc index a65789841..60c8c039e 100644 --- a/eval/eval/jump_step.cc +++ b/eval/eval/jump_step.cc @@ -14,14 +14,15 @@ #include "eval/eval/jump_step.h" +#include #include #include #include #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "absl/types/optional.h" #include "common/value.h" +#include "eval/eval/evaluator_core.h" #include "eval/internal/errors.h" namespace google::api::expr::runtime { @@ -47,28 +48,24 @@ class JumpStep : public JumpStepBase { class CondJumpStep : public JumpStepBase { public: - // Constructs FunctionStep that uses overloads specified. - CondJumpStep(bool jump_condition, bool leave_on_stack, - absl::optional jump_offset, int64_t expr_id) + CondJumpStep(bool jump_condition, absl::optional jump_offset, + size_t stack_size, int64_t expr_id) : JumpStepBase(jump_offset, expr_id), jump_condition_(jump_condition), - leave_on_stack_(leave_on_stack) {} + stack_size_(stack_size) {} absl::Status Evaluate(ExecutionFrame* frame) const override { // Peek the top value - if (!frame->value_stack().HasEnough(1)) { + if (!frame->value_stack().HasEnough(stack_size_)) { return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); } const auto& value = frame->value_stack().Peek(); - const auto should_jump = value.Is() && - jump_condition_ == value.GetBool().NativeValue(); - - if (!leave_on_stack_) { - frame->value_stack().Pop(1); - } + const auto should_jump = + value.IsBool() && jump_condition_ == value.GetBool().NativeValue(); if (should_jump) { + frame->value_stack().SwapAndPop(stack_size_, stack_size_ - 1); return Jump(frame); } @@ -77,7 +74,31 @@ class CondJumpStep : public JumpStepBase { private: const bool jump_condition_; - const bool leave_on_stack_; + const size_t stack_size_; +}; + +class TernaryCondJumpStep : public JumpStepBase { + public: + TernaryCondJumpStep(absl::optional jump_offset, int64_t expr_id) + : JumpStepBase(jump_offset, expr_id) {} + + absl::Status Evaluate(ExecutionFrame* frame) const override { + // Peek the top value + if (!frame->value_stack().HasEnough(1)) { + return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + } + + const auto& value = frame->value_stack().Peek(); + const auto should_jump = value.IsBool() && !value.GetBool().NativeValue(); + + frame->value_stack().Pop(1); + + if (should_jump) { + return Jump(frame); + } + + return absl::OkStatus(); + } }; class BoolCheckJumpStep : public JumpStepBase { @@ -121,13 +142,16 @@ class BoolCheckJumpStep : public JumpStepBase { } // namespace // Factory method for Conditional Jump step. -// Conditional Jump requires a boolean value to sit on the stack. -// It is compared to jump_condition, and if matched, jump is performed. std::unique_ptr CreateCondJumpStep( - bool jump_condition, bool leave_on_stack, absl::optional jump_offset, - int64_t expr_id) { - return std::make_unique(jump_condition, leave_on_stack, - jump_offset, expr_id); + bool jump_condition, absl::optional jump_offset, int64_t expr_id) { + return std::make_unique(jump_condition, jump_offset, 1, + expr_id); +} + +// Factory method for Ternary Conditional Jump step. +std::unique_ptr CreateTernaryCondJumpStep( + absl::optional jump_offset, int64_t expr_id) { + return std::make_unique(jump_offset, expr_id); } // Factory method for Jump step. diff --git a/eval/eval/jump_step.h b/eval/eval/jump_step.h index 55147da5f..27deba352 100644 --- a/eval/eval/jump_step.h +++ b/eval/eval/jump_step.h @@ -48,14 +48,19 @@ class JumpStepBase : public ExpressionStepBase { std::unique_ptr CreateJumpStep(absl::optional jump_offset, int64_t expr_id); -// Factory method for Conditional Jump step. +// Factory method for Conditional Jump step (used for and/or shortcircuiting). // Conditional Jump requires a boolean value to sit on the stack. // It is compared to jump_condition, and if matched, jump is performed. -// leave on stack indicates whether value should be kept on top of the stack or -// removed. +// The boolean value is left on top of the stack. std::unique_ptr CreateCondJumpStep( - bool jump_condition, bool leave_on_stack, absl::optional jump_offset, - int64_t expr_id); + bool jump_condition, absl::optional jump_offset, int64_t expr_id); + +// Factory method for Ternary Conditional Jump step. +// Requires a boolean condition value on top of the stack. +// If the boolean value is false, a jump is performed to the second branch. +// The condition value is popped from the stack before jumping or continuing. +std::unique_ptr CreateTernaryCondJumpStep( + absl::optional jump_offset, int64_t expr_id); // Factory method for ErrorJump step. // This step performs a Jump when an Error is on the top of the stack. diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index f844d8c05..6d5c8bbb3 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "absl/status/status.h" @@ -188,8 +189,8 @@ absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, class LogicalOpStep : public ExpressionStepBase { public: // Constructs FunctionStep that uses overloads specified. - LogicalOpStep(OpType op_type, int64_t expr_id) - : ExpressionStepBase(expr_id), op_type_(op_type) { + LogicalOpStep(OpType op_type, size_t count, int64_t expr_id) + : ExpressionStepBase(expr_id), op_type_(op_type), count_(count) { shortcircuit_ = (op_type_ == OpType::kOr); } @@ -198,28 +199,25 @@ class LogicalOpStep : public ExpressionStepBase { private: void Calculate(ExecutionFrame* frame, absl::Span args, Value& result) const { - bool bool_args[2]; - bool has_bool_args[2]; + std::optional error_pos; for (size_t i = 0; i < args.size(); i++) { - has_bool_args[i] = args[i]->Is(); - if (has_bool_args[i]) { - bool_args[i] = args[i].GetBool().NativeValue(); - if (bool_args[i] == shortcircuit_) { - result = BoolValue{bool_args[i]}; - return; - } - } - } - - if (has_bool_args[0] && has_bool_args[1]) { - switch (op_type_) { - case OpType::kAnd: - result = BoolValue{bool_args[0] && bool_args[1]}; - return; - case OpType::kOr: - result = BoolValue{bool_args[0] || bool_args[1]}; - return; + const Value& arg = args[i]; + switch (arg.kind()) { + case ValueKind::kBool: + if (arg.GetBool() == shortcircuit_) { + result = arg; + return; + } + break; + case ValueKind::kUnknown: + break; + case ValueKind::kError: + default: + if (!error_pos.has_value()) { + error_pos = i; + } + break; } } @@ -237,31 +235,31 @@ class LogicalOpStep : public ExpressionStepBase { } } - if (args[0]->Is()) { - result = args[0]; - return; - } else if (args[1]->Is()) { - result = args[1]; + if (!error_pos.has_value()) { + result = cel::BoolValue(!shortcircuit_); return; } - // Fallback. - result = cel::ErrorValue(CreateNoMatchingOverloadError( - (op_type_ == OpType::kOr) ? cel::builtin::kOr : cel::builtin::kAnd)); + result = args[error_pos.value()]; + if (!result.IsError()) { + result = cel::ErrorValue(CreateNoMatchingOverloadError( + (op_type_ == OpType::kOr) ? cel::builtin::kOr : cel::builtin::kAnd)); + } } const OpType op_type_; + size_t count_; bool shortcircuit_; }; absl::Status LogicalOpStep::Evaluate(ExecutionFrame* frame) const { // Must have 2 or more values on the stack. - if (!frame->value_stack().HasEnough(2)) { + if (!frame->value_stack().HasEnough(count_)) { return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); } // Create Span object that contains input arguments to the function. - auto args = frame->value_stack().GetSpan(2); + auto args = frame->value_stack().GetSpan(count_); Value result; Calculate(frame, args, result); frame->value_stack().PopAndPush(args.size(), std::move(result)); @@ -453,12 +451,12 @@ std::unique_ptr CreateDirectOrStep( // Factory method for "And" Execution step absl::StatusOr> CreateAndStep(int64_t expr_id) { - return std::make_unique(OpType::kAnd, expr_id); + return std::make_unique(OpType::kAnd, 2, expr_id); } // Factory method for "Or" Execution step absl::StatusOr> CreateOrStep(int64_t expr_id) { - return std::make_unique(OpType::kOr, expr_id); + return std::make_unique(OpType::kOr, 2, expr_id); } // Factory method for recursive logical not "!" Execution step From de51eee5b19348c38164356d84cef3c73e03f9af Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 10 Jul 2026 10:24:30 -0700 Subject: [PATCH 114/350] Move bind proto to activation utility to /runtime. PiperOrigin-RevId: 945776679 --- conformance/policy/BUILD | 2 +- conformance/policy/policy_conformance_test.cc | 7 +- extensions/protobuf/BUILD | 3 +- .../protobuf/bind_proto_to_activation.cc | 77 ------ .../protobuf/bind_proto_to_activation.h | 109 +------- runtime/BUILD | 36 +++ runtime/bind_proto_to_activation.cc | 92 +++++++ runtime/bind_proto_to_activation.h | 127 +++++++++ runtime/bind_proto_to_activation_test.cc | 244 ++++++++++++++++++ 9 files changed, 513 insertions(+), 184 deletions(-) create mode 100644 runtime/bind_proto_to_activation.cc create mode 100644 runtime/bind_proto_to_activation.h create mode 100644 runtime/bind_proto_to_activation_test.cc diff --git a/conformance/policy/BUILD b/conformance/policy/BUILD index e5d503d88..42adbcf90 100644 --- a/conformance/policy/BUILD +++ b/conformance/policy/BUILD @@ -36,7 +36,6 @@ cc_library( "//env:env_std_extensions", "//env:env_yaml", "//env:runtime_std_extensions", - "//extensions/protobuf:bind_proto_to_activation", "//extensions/protobuf:enum_adapter", "//internal:runfiles", "//internal:status_macros", @@ -50,6 +49,7 @@ cc_library( "//policy:yaml_policy_parser", "//runtime", "//runtime:activation", + "//runtime:bind_proto_to_activation", "//runtime:function_adapter", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/log:absl_check", diff --git a/conformance/policy/policy_conformance_test.cc b/conformance/policy/policy_conformance_test.cc index ef53ab34e..53fbcb419 100644 --- a/conformance/policy/policy_conformance_test.cc +++ b/conformance/policy/policy_conformance_test.cc @@ -40,7 +40,6 @@ #include "env/env_std_extensions.h" #include "env/env_yaml.h" #include "env/runtime_std_extensions.h" -#include "extensions/protobuf/bind_proto_to_activation.h" #include "extensions/protobuf/enum_adapter.h" #include "internal/runfiles.h" #include "internal/status_macros.h" @@ -53,6 +52,7 @@ #include "policy/test_util.h" #include "policy/yaml_policy_parser.h" #include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" #include "runtime/function_adapter.h" #include "runtime/runtime.h" #include "cel/expr/conformance/test/suite.pb.h" @@ -384,9 +384,8 @@ absl::Status PopulateActivation( "Failed to resolve context message for test case"); } - return cel::extensions::BindProtoToActivation( - *context_message, - cel::extensions::BindProtoUnsetFieldBehavior::kBindDefaultValue, + return cel::BindProtoToActivation( + *context_message, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, descriptor_pool, message_factory, arena, &activation); } diff --git a/extensions/protobuf/BUILD b/extensions/protobuf/BUILD index 3f4081b09..d814057e7 100644 --- a/extensions/protobuf/BUILD +++ b/extensions/protobuf/BUILD @@ -165,11 +165,12 @@ cc_library( srcs = ["bind_proto_to_activation.cc"], hdrs = ["bind_proto_to_activation.h"], deps = [ - ":value", "//common:casting", "//common:value", "//internal:status_macros", "//runtime:activation", + "//runtime:bind_proto_to_activation", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/extensions/protobuf/bind_proto_to_activation.cc b/extensions/protobuf/bind_proto_to_activation.cc index aa151cb85..bdebd08eb 100644 --- a/extensions/protobuf/bind_proto_to_activation.cc +++ b/extensions/protobuf/bind_proto_to_activation.cc @@ -13,80 +13,3 @@ // limitations under the License. #include "extensions/protobuf/bind_proto_to_activation.h" - -#include - -#include "absl/base/nullability.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "common/value.h" -#include "internal/status_macros.h" -#include "runtime/activation.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/message.h" - -namespace cel::extensions::protobuf_internal { - -namespace { - -using ::google::protobuf::Descriptor; - -absl::StatusOr ShouldBindField( - const google::protobuf::FieldDescriptor* field_desc, const StructValue& struct_value, - BindProtoUnsetFieldBehavior unset_field_behavior) { - if (unset_field_behavior == BindProtoUnsetFieldBehavior::kBindDefaultValue || - field_desc->is_repeated()) { - return true; - } - return struct_value.HasFieldByNumber(field_desc->number()); -} - -absl::StatusOr GetFieldValue( - const google::protobuf::FieldDescriptor* field_desc, const StructValue& struct_value, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) { - // Special case unset any. - if (field_desc->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && - field_desc->message_type()->well_known_type() == - Descriptor::WELLKNOWNTYPE_ANY) { - CEL_ASSIGN_OR_RETURN(bool present, - struct_value.HasFieldByNumber(field_desc->number())); - if (!present) { - return NullValue(); - } - } - - return struct_value.GetFieldByNumber(field_desc->number(), descriptor_pool, - message_factory, arena); -} - -} // namespace - -absl::Status BindProtoToActivation( - const Descriptor& descriptor, const StructValue& struct_value, - BindProtoUnsetFieldBehavior unset_field_behavior, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { - for (int i = 0; i < descriptor.field_count(); i++) { - const google::protobuf::FieldDescriptor* field_desc = descriptor.field(i); - CEL_ASSIGN_OR_RETURN( - bool should_bind, - ShouldBindField(field_desc, struct_value, unset_field_behavior)); - if (!should_bind) { - continue; - } - - CEL_ASSIGN_OR_RETURN( - Value field, GetFieldValue(field_desc, struct_value, descriptor_pool, - message_factory, arena)); - - activation->InsertOrAssignValue(field_desc->name(), std::move(field)); - } - - return absl::OkStatus(); -} - -} // namespace cel::extensions::protobuf_internal diff --git a/extensions/protobuf/bind_proto_to_activation.h b/extensions/protobuf/bind_proto_to_activation.h index 61f43c13d..97dfc6c0c 100644 --- a/extensions/protobuf/bind_proto_to_activation.h +++ b/extensions/protobuf/bind_proto_to_activation.h @@ -15,116 +15,23 @@ #ifndef THIRD_PARTY_CEL_CPP_EXTENSIONS_PROTOBUF_BIND_PROTO_TO_ACTIVATION_H_ #define THIRD_PARTY_CEL_CPP_EXTENSIONS_PROTOBUF_BIND_PROTO_TO_ACTIVATION_H_ -#include - -#include "absl/base/nullability.h" -#include "absl/status/status.h" -#include "common/casting.h" -#include "common/value.h" -#include "extensions/protobuf/value.h" -#include "internal/status_macros.h" -#include "runtime/activation.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/message.h" +#include "absl/base/attributes.h" +#include "runtime/bind_proto_to_activation.h" namespace cel::extensions { -// Option for handling unset fields on the context proto. -enum class BindProtoUnsetFieldBehavior { - // Bind the message defined default or zero value. - kBindDefaultValue, - // Skip binding unset fields, no value is bound for the corresponding - // variable. - kSkip -}; +using BindProtoUnsetFieldBehavior ABSL_DEPRECATED( + "Use cel::BindProtoUnsetFieldBehavior instead") = + ::cel::BindProtoUnsetFieldBehavior; + +using ::cel::BindProtoToActivation; namespace protobuf_internal { -// Implements binding provided the context message has already -// been adapted to a suitable struct value. -absl::Status BindProtoToActivation( - const google::protobuf::Descriptor& descriptor, const StructValue& struct_value, - BindProtoUnsetFieldBehavior unset_field_behavior, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation); +using ::cel::runtime_internal::BindProtoToActivation; } // namespace protobuf_internal -// Utility method, that takes a protobuf Message and interprets it as a -// namespace, binding its fields to Activation. This is often referred to as a -// context message. -// -// Field names and values become respective names and values of parameters -// bound to the Activation object. -// Example: -// Assume we have a protobuf message of type: -// message Person { -// int age = 1; -// string name = 2; -// } -// -// The sample code snippet will look as follows: -// -// Person person; -// person.set_name("John Doe"); -// person.age(42); -// -// CEL_RETURN_IF_ERROR(BindProtoToActivation(person, value_factory, -// activation)); -// -// After this snippet, activation will have two parameters bound: -// "name", with string value of "John Doe" -// "age", with int value of 42. -// -// The default behavior for unset fields is to skip them. E.g. if the name field -// is not set on the Person message, it will not be bound in to the activation. -// BindProtoUnsetFieldBehavior::kBindDefault, will bind the cc proto api default -// for the field (either an explicit default value or a type specific default). -// -// For repeated fields, an unset field is bound as an empty list. -template -absl::Status BindProtoToActivation( - const T& context, BindProtoUnsetFieldBehavior unset_field_behavior, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { - static_assert(std::is_base_of_v); - // TODO(uncreated-issue/68): for simplicity, just convert the whole message to a - // struct value. For performance, may be better to convert members as needed. - CEL_ASSIGN_OR_RETURN( - Value parent, - ProtoMessageToValue(context, descriptor_pool, message_factory, arena)); - - if (!InstanceOf(parent)) { - return absl::InvalidArgumentError( - absl::StrCat("context is a well-known type: ", context.GetTypeName())); - } - const StructValue& struct_value = Cast(parent); - - const google::protobuf::Descriptor* descriptor = context.GetDescriptor(); - - if (descriptor == nullptr) { - return absl::InvalidArgumentError( - absl::StrCat("context missing descriptor: ", context.GetTypeName())); - } - - return protobuf_internal::BindProtoToActivation( - *descriptor, struct_value, unset_field_behavior, descriptor_pool, - message_factory, arena, activation); -} -template -absl::Status BindProtoToActivation( - const T& context, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { - return BindProtoToActivation(context, BindProtoUnsetFieldBehavior::kSkip, - descriptor_pool, message_factory, arena, - activation); -} - } // namespace cel::extensions #endif // THIRD_PARTY_CEL_CPP_EXTENSIONS_PROTOBUF_BIND_PROTO_TO_ACTIVATION_H_ diff --git a/runtime/BUILD b/runtime/BUILD index 34ff411a1..da2192439 100644 --- a/runtime/BUILD +++ b/runtime/BUILD @@ -110,6 +110,42 @@ cc_test( ], ) +cc_library( + name = "bind_proto_to_activation", + srcs = ["bind_proto_to_activation.cc"], + hdrs = ["bind_proto_to_activation.h"], + deps = [ + ":activation", + "//common:casting", + "//common:value", + "//internal:status_macros", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "bind_proto_to_activation_test", + srcs = ["bind_proto_to_activation_test.cc"], + deps = [ + ":activation", + ":bind_proto_to_activation", + "//common:casting", + "//common:value", + "//common:value_testing", + "//internal:testing", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/types:optional", + "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:wrappers_cc_proto", + ], +) + cc_library( name = "register_function_helper", hdrs = ["register_function_helper.h"], diff --git a/runtime/bind_proto_to_activation.cc b/runtime/bind_proto_to_activation.cc new file mode 100644 index 000000000..011feea7c --- /dev/null +++ b/runtime/bind_proto_to_activation.cc @@ -0,0 +1,92 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/bind_proto_to_activation.h" + +#include + +#include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "common/value.h" +#include "internal/status_macros.h" +#include "runtime/activation.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace cel::runtime_internal { + +namespace { + +using ::google::protobuf::Descriptor; + +absl::StatusOr ShouldBindField( + const google::protobuf::FieldDescriptor* field_desc, const StructValue& struct_value, + BindProtoUnsetFieldBehavior unset_field_behavior) { + if (unset_field_behavior == BindProtoUnsetFieldBehavior::kBindDefaultValue || + field_desc->is_repeated()) { + return true; + } + return struct_value.HasFieldByNumber(field_desc->number()); +} + +absl::StatusOr GetFieldValue( + const google::protobuf::FieldDescriptor* field_desc, const StructValue& struct_value, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { + // Special case unset any. + if (field_desc->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field_desc->message_type()->well_known_type() == + Descriptor::WELLKNOWNTYPE_ANY) { + CEL_ASSIGN_OR_RETURN(bool present, + struct_value.HasFieldByNumber(field_desc->number())); + if (!present) { + return NullValue(); + } + } + + return struct_value.GetFieldByNumber(field_desc->number(), descriptor_pool, + message_factory, arena); +} + +} // namespace + +absl::Status BindProtoToActivation( + const Descriptor& descriptor, const StructValue& struct_value, + BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + for (int i = 0; i < descriptor.field_count(); i++) { + const google::protobuf::FieldDescriptor* field_desc = descriptor.field(i); + CEL_ASSIGN_OR_RETURN( + bool should_bind, + ShouldBindField(field_desc, struct_value, unset_field_behavior)); + if (!should_bind) { + continue; + } + + CEL_ASSIGN_OR_RETURN( + Value field, GetFieldValue(field_desc, struct_value, descriptor_pool, + message_factory, arena)); + + activation->InsertOrAssignValue(field_desc->name(), std::move(field)); + } + + return absl::OkStatus(); +} + +} // namespace cel::runtime_internal diff --git a/runtime/bind_proto_to_activation.h b/runtime/bind_proto_to_activation.h new file mode 100644 index 000000000..9bc866937 --- /dev/null +++ b/runtime/bind_proto_to_activation.h @@ -0,0 +1,127 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_RUNTIME_BIND_PROTO_TO_ACTIVATION_H_ +#define THIRD_PARTY_CEL_CPP_RUNTIME_BIND_PROTO_TO_ACTIVATION_H_ + +#include + +#include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "common/casting.h" +#include "common/value.h" +#include "runtime/activation.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace cel { + +// Option for handling unset fields on the context proto. +enum class BindProtoUnsetFieldBehavior { + // Bind the message defined default or zero value. + kBindDefaultValue, + // Skip binding unset fields, no value is bound for the corresponding + // variable. + kSkip +}; + +namespace runtime_internal { + +// Implements binding provided the context message has already +// been adapted to a suitable struct value. +absl::Status BindProtoToActivation( + const google::protobuf::Descriptor& descriptor, const StructValue& struct_value, + BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation); + +} // namespace runtime_internal + +// Utility method, that takes a protobuf Message and interprets it as a +// namespace, binding its fields to Activation. This is often referred to as a +// context message. +// +// Field names and values become respective names and values of parameters +// bound to the Activation object. +// Example: +// Assume we have a protobuf message of type: +// message Person { +// int age = 1; +// string name = 2; +// } +// +// The sample code snippet will look as follows: +// +// Person person; +// person.set_name("John Doe"); +// person.age(42); +// +// CEL_RETURN_IF_ERROR(BindProtoToActivation(person, value_factory, +// activation)); +// +// After this snippet, activation will have two parameters bound: +// "name", with string value of "John Doe" +// "age", with int value of 42. +// +// The default behavior for unset fields is to skip them. E.g. if the name field +// is not set on the Person message, it will not be bound in to the activation. +// BindProtoUnsetFieldBehavior::kBindDefault, will bind the cc proto api default +// for the field (either an explicit default value or a type specific default). +// +// For repeated fields, an unset field is bound as an empty list. +template +absl::Status BindProtoToActivation( + const T& context, BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + static_assert(std::is_base_of_v); + Value parent = + Value::FromMessage(context, descriptor_pool, message_factory, arena); + + if (!InstanceOf(parent)) { + return absl::InvalidArgumentError( + absl::StrCat("context is a well-known type: ", context.GetTypeName())); + } + const StructValue& struct_value = Cast(parent); + + const google::protobuf::Descriptor* descriptor = context.GetDescriptor(); + + if (descriptor == nullptr) { + return absl::InvalidArgumentError( + absl::StrCat("context missing descriptor: ", context.GetTypeName())); + } + + return runtime_internal::BindProtoToActivation( + *descriptor, struct_value, unset_field_behavior, descriptor_pool, + message_factory, arena, activation); +} + +template +absl::Status BindProtoToActivation( + const T& context, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + return BindProtoToActivation(context, BindProtoUnsetFieldBehavior::kSkip, + descriptor_pool, message_factory, arena, + activation); +} + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_RUNTIME_BIND_PROTO_TO_ACTIVATION_H_ diff --git a/runtime/bind_proto_to_activation_test.cc b/runtime/bind_proto_to_activation_test.cc new file mode 100644 index 000000000..2242f47c0 --- /dev/null +++ b/runtime/bind_proto_to_activation_test.cc @@ -0,0 +1,244 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/bind_proto_to_activation.h" + +#include "google/protobuf/wrappers.pb.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/types/optional.h" +#include "common/casting.h" +#include "common/value.h" +#include "common/value_testing.h" +#include "internal/testing.h" +#include "runtime/activation.h" +#include "cel/expr/conformance/proto2/test_all_types.pb.h" +#include "google/protobuf/arena.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::cel::expr::conformance::proto2::TestAllTypes; +using ::cel::test::IntValueIs; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::Optional; + +using BindProtoToActivationTest = common_internal::ValueTest<>; + +TEST_F(BindProtoToActivationTest, BindProtoToActivation) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_int64", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(123)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationWktUnsupported) { + google::protobuf::Int64Value int64_value; + int64_value.set_value(123); + Activation activation; + + EXPECT_THAT(BindProtoToActivation(int64_value, descriptor_pool(), + message_factory(), arena(), &activation), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("google.protobuf.Int64Value"))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationSkip) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(activation.FindVariable("single_sint32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationDefault) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT( + BindProtoToActivation( + test_all_types, BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool(), message_factory(), arena(), &activation), + IsOk()); + + // from test_all_types.proto + // optional int32 single_int32 = 1 [default = -32]; + EXPECT_THAT(activation.FindVariable("single_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(-32)))); + EXPECT_THAT(activation.FindVariable("single_sint32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(0)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationDefaultAny) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT( + BindProtoToActivation( + test_all_types, BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool(), message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_any", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(test::IsNullValue()))); +} + +MATCHER_P(IsListValueOfSize, size, "") { + const Value& v = arg; + + auto value = As(v); + if (!value) { + return false; + } + auto s = value->Size(); + return s.ok() && *s == size; +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeated) { + TestAllTypes test_all_types; + test_all_types.add_repeated_int64(123); + test_all_types.add_repeated_int64(456); + test_all_types.add_repeated_int64(789); + + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("repeated_int64", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsListValueOfSize(3)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeatedEmpty) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("repeated_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsListValueOfSize(0)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeatedComplex) { + TestAllTypes test_all_types; + auto* nested = test_all_types.add_repeated_nested_message(); + nested->set_bb(123); + nested = test_all_types.add_repeated_nested_message(); + nested->set_bb(456); + nested = test_all_types.add_repeated_nested_message(); + nested->set_bb(789); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT( + activation.FindVariable("repeated_nested_message", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsListValueOfSize(3)))); +} + +MATCHER_P(IsMapValueOfSize, size, "") { + const Value& v = arg; + + auto value = As(v); + if (!value) { + return false; + } + auto s = value->Size(); + return s.ok() && *s == size; +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationMap) { + TestAllTypes test_all_types; + (*test_all_types.mutable_map_int64_int64())[1] = 2; + (*test_all_types.mutable_map_int64_int64())[2] = 4; + + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("map_int64_int64", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsMapValueOfSize(2)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationMapEmpty) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("map_int32_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsMapValueOfSize(0)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoToActivationMapComplex) { + TestAllTypes test_all_types; + TestAllTypes::NestedMessage value; + value.set_bb(42); + (*test_all_types.mutable_map_int64_message())[1] = value; + (*test_all_types.mutable_map_int64_message())[2] = value; + + Activation activation; + + ASSERT_THAT(BindProtoToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("map_int64_message", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IsMapValueOfSize(2)))); +} + +} // namespace +} // namespace cel From 4c95b4f45eec9efd46a3812f08aadd6c2880909a Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 10 Jul 2026 10:36:55 -0700 Subject: [PATCH 115/350] Refactor short-circuit planning logic. Split out specializations for optional or and &&/||. They are less uniform after introducing variadics. PiperOrigin-RevId: 945783246 --- eval/compiler/flat_expr_builder.cc | 206 ++++++++++++----------------- 1 file changed, 81 insertions(+), 125 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 9cb9fbb01..fda57ca73 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -198,14 +198,7 @@ class CondVisitor { virtual void PostVisitTarget(const cel::Expr* expr) {} }; -enum class BinaryCond { - kAnd = 0, - kOr, - kOptionalOr, - kOptionalOrValue, -}; - -// Visitor managing the "&&" and "||" operatiions. +// Visitor managing the "&&" and "||" (boolean logic) operations. // Implements short-circuiting if enabled. // // With short-circuiting enabled, generates a program like: @@ -218,20 +211,41 @@ enum class BinaryCond { // | i + 3 | BooleanOperator | Op(arg1, arg2) | // | i + 4 | | arg1 | Op(arg1, arg2) | // +-------------+------------------------+------------------------+ -class BinaryCondVisitor : public CondVisitor { +class LogicalCondVisitor : public CondVisitor { public: - explicit BinaryCondVisitor(FlatExprVisitor* visitor, BinaryCond cond, - bool short_circuiting) - : visitor_(visitor), cond_(cond), short_circuiting_(short_circuiting) {} + explicit LogicalCondVisitor(FlatExprVisitor* visitor, bool is_or, + bool short_circuiting) + : visitor_(visitor), is_or_(is_or), short_circuiting_(short_circuiting) {} void PreVisit(const cel::Expr* expr) override; void PostVisitArg(int arg_num, const cel::Expr* expr) override; void PostVisit(const cel::Expr* expr) override; + + private: + FlatExprVisitor* visitor_; + const bool is_or_; + std::vector jump_steps_; + bool short_circuiting_; +}; + +// Visitor managing optional "or" and "orValue" operations. +// Implements short-circuiting if enabled. +class OptionalOrCondVisitor : public CondVisitor { + public: + explicit OptionalOrCondVisitor(FlatExprVisitor* visitor, bool is_or_value, + bool short_circuiting) + : visitor_(visitor), + is_or_value_(is_or_value), + short_circuiting_(short_circuiting) {} + + void PreVisit(const cel::Expr* expr) override; + void PostVisitArg(int arg_num, const cel::Expr* expr) override {} void PostVisitTarget(const cel::Expr* expr) override; + void PostVisit(const cel::Expr* expr) override; private: FlatExprVisitor* visitor_; - const BinaryCond cond_; + const bool is_or_value_; std::vector jump_steps_; bool short_circuiting_; }; @@ -997,11 +1011,11 @@ class FlatExprVisitor : public cel::AstVisitor { std::unique_ptr cond_visitor; if (call_expr.function() == cel::builtin::kAnd) { - cond_visitor = std::make_unique( - this, BinaryCond::kAnd, options_.short_circuiting); + cond_visitor = std::make_unique( + this, /*is_or=*/false, options_.short_circuiting); } else if (call_expr.function() == cel::builtin::kOr) { - cond_visitor = std::make_unique( - this, BinaryCond::kOr, options_.short_circuiting); + cond_visitor = std::make_unique( + this, /*is_or=*/true, options_.short_circuiting); } else if (call_expr.function() == cel::builtin::kTernary) { if (options_.short_circuiting) { cond_visitor = std::make_unique(this); @@ -1011,13 +1025,13 @@ class FlatExprVisitor : public cel::AstVisitor { } else if (enable_optional_types_ && call_expr.function() == kOptionalOrFn && call_expr.has_target() && call_expr.args().size() == 1) { - cond_visitor = std::make_unique( - this, BinaryCond::kOptionalOr, options_.short_circuiting); + cond_visitor = std::make_unique( + this, /*is_or_value=*/false, options_.short_circuiting); } else if (enable_optional_types_ && call_expr.function() == kOptionalOrValueFn && call_expr.has_target() && call_expr.args().size() == 1) { - cond_visitor = std::make_unique( - this, BinaryCond::kOptionalOrValue, options_.short_circuiting); + cond_visitor = std::make_unique( + this, /*is_or_value=*/true, options_.short_circuiting); } else if (IsBlock(&call_expr)) { // cel.@block if (block_.has_value()) { @@ -2147,91 +2161,64 @@ FlatExprVisitor::HandleHeterogeneousEqualityIn(const cel::Expr& expr, return CallHandlerResult::kIntercepted; } -void BinaryCondVisitor::PreVisit(const cel::Expr* expr) { - switch (cond_) { - case BinaryCond::kAnd: - ABSL_FALLTHROUGH_INTENDED; - case BinaryCond::kOr: - visitor_->ValidateOrError( - !expr->call_expr().has_target() && - expr->call_expr().args().size() >= 2, - "Invalid argument count for a binary function call."); - break; - case BinaryCond::kOptionalOr: - ABSL_FALLTHROUGH_INTENDED; - case BinaryCond::kOptionalOrValue: - visitor_->ValidateOrError(expr->call_expr().has_target() && - expr->call_expr().args().size() == 1, - "Invalid argument count for or/orValue call."); - break; - } +void LogicalCondVisitor::PreVisit(const cel::Expr* expr) { + visitor_->ValidateOrError( + !expr->call_expr().has_target() && expr->call_expr().args().size() >= 2, + "Invalid argument count for a binary function call."); } -void BinaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { +void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { if (visitor_->PlanRecursiveProgram()) { return; } const int last_arg_index = expr->call_expr().args().size() - 1; - if (cond_ == BinaryCond::kAnd || cond_ == BinaryCond::kOr) { - if (arg_num > 0) { - switch (cond_) { - case BinaryCond::kAnd: - visitor_->AddStep(CreateAndStep(expr->id())); - break; - case BinaryCond::kOr: - visitor_->AddStep(CreateOrStep(expr->id())); - break; - default: - break; - } - if (short_circuiting_ && !jump_steps_.empty()) { - visitor_->SetProgressStatusIfError( - jump_steps_.back().set_target(visitor_->GetCurrentIndex())); - } + if (arg_num > 0) { + if (is_or_) { + visitor_->AddStep(CreateOrStep(expr->id())); + } else { + visitor_->AddStep(CreateAndStep(expr->id())); } - if (short_circuiting_ && arg_num < last_arg_index) { - std::unique_ptr jump_step; - switch (cond_) { - case BinaryCond::kAnd: - jump_step = CreateCondJumpStep(false, {}, expr->id()); - break; - case BinaryCond::kOr: - jump_step = CreateCondJumpStep(true, {}, expr->id()); - break; - default: - ABSL_UNREACHABLE(); - } - ProgramStepIndex index = visitor_->GetCurrentIndex(); - if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); - jump_step_ptr) { - jump_steps_.push_back(Jump(index, jump_step_ptr)); - } + if (short_circuiting_ && !jump_steps_.empty()) { + visitor_->SetProgressStatusIfError( + jump_steps_.back().set_target(visitor_->GetCurrentIndex())); } } + if (short_circuiting_ && arg_num < last_arg_index) { + std::unique_ptr jump_step = + is_or_ ? CreateCondJumpStep(true, {}, expr->id()) + : CreateCondJumpStep(false, {}, expr->id()); + ProgramStepIndex index = visitor_->GetCurrentIndex(); + if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); + jump_step_ptr) { + jump_steps_.push_back(Jump(index, jump_step_ptr)); + } + } +} + +void LogicalCondVisitor::PostVisit(const cel::Expr* expr) { + if (visitor_->PlanRecursiveProgram()) { + visitor_->MakeShortcircuitRecursive(expr, is_or_); + } +} + +void OptionalOrCondVisitor::PreVisit(const cel::Expr* expr) { + visitor_->ValidateOrError( + expr->call_expr().has_target() && expr->call_expr().args().size() == 1, + "Invalid argument count for or/orValue call."); } -void BinaryCondVisitor::PostVisitTarget(const cel::Expr* expr) { +void OptionalOrCondVisitor::PostVisitTarget(const cel::Expr* expr) { if (visitor_->PlanRecursiveProgram()) { return; } - if (short_circuiting_ && (cond_ == BinaryCond::kOptionalOr || - cond_ == BinaryCond::kOptionalOrValue)) { + if (short_circuiting_) { // If first branch evaluation result is enough to determine output, // jump over the second branch and provide result of the first argument as // final output. // Retain a pointer to the jump step so we can update the target after // planning the second argument. - std::unique_ptr jump_step; - switch (cond_) { - case BinaryCond::kOptionalOr: - jump_step = CreateOptionalHasValueJumpStep(false, expr->id()); - break; - case BinaryCond::kOptionalOrValue: - jump_step = CreateOptionalHasValueJumpStep(true, expr->id()); - break; - default: - ABSL_UNREACHABLE(); - } + std::unique_ptr jump_step = + CreateOptionalHasValueJumpStep(is_or_value_, expr->id()); ProgramStepIndex index = visitor_->GetCurrentIndex(); if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); jump_step_ptr) { @@ -2240,48 +2227,17 @@ void BinaryCondVisitor::PostVisitTarget(const cel::Expr* expr) { } } -void BinaryCondVisitor::PostVisit(const cel::Expr* expr) { +void OptionalOrCondVisitor::PostVisit(const cel::Expr* expr) { if (visitor_->PlanRecursiveProgram()) { - switch (cond_) { - case BinaryCond::kAnd: - visitor_->MakeShortcircuitRecursive(expr, /*is_or=*/false); - break; - case BinaryCond::kOr: - visitor_->MakeShortcircuitRecursive(expr, /*is_or=*/true); - break; - case BinaryCond::kOptionalOr: - visitor_->MakeOptionalShortcircuit(expr, - /*is_or_value=*/false); - break; - case BinaryCond::kOptionalOrValue: - visitor_->MakeOptionalShortcircuit(expr, - /*is_or_value=*/true); - break; - default: - ABSL_UNREACHABLE(); - } + visitor_->MakeOptionalShortcircuit(expr, is_or_value_); return; } - if (cond_ == BinaryCond::kOptionalOr || - cond_ == BinaryCond::kOptionalOrValue) { - switch (cond_) { - case BinaryCond::kOptionalOr: - visitor_->AddStep( - CreateOptionalOrStep(/*is_or_value=*/false, expr->id())); - break; - case BinaryCond::kOptionalOrValue: - visitor_->AddStep( - CreateOptionalOrStep(/*is_or_value=*/true, expr->id())); - break; - default: - ABSL_UNREACHABLE(); - } - if (short_circuiting_) { - for (auto& jump : jump_steps_) { - visitor_->SetProgressStatusIfError( - jump.set_target(visitor_->GetCurrentIndex())); - } + visitor_->AddStep(CreateOptionalOrStep(is_or_value_, expr->id())); + if (short_circuiting_) { + for (auto& jump : jump_steps_) { + visitor_->SetProgressStatusIfError( + jump.set_target(visitor_->GetCurrentIndex())); } } } From 05556a07a062b8063d34086850b23f3b415c0fe1 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 10 Jul 2026 11:03:21 -0700 Subject: [PATCH 116/350] Add wrapping version of BindProtoToActivation. PiperOrigin-RevId: 945797968 --- .../protobuf/bind_proto_to_activation.h | 1 + runtime/BUILD | 2 +- runtime/bind_proto_to_activation.h | 89 ++++++++++++++----- runtime/bind_proto_to_activation_test.cc | 34 +++++++ 4 files changed, 104 insertions(+), 22 deletions(-) diff --git a/extensions/protobuf/bind_proto_to_activation.h b/extensions/protobuf/bind_proto_to_activation.h index 97dfc6c0c..93025185b 100644 --- a/extensions/protobuf/bind_proto_to_activation.h +++ b/extensions/protobuf/bind_proto_to_activation.h @@ -25,6 +25,7 @@ using BindProtoUnsetFieldBehavior ABSL_DEPRECATED( ::cel::BindProtoUnsetFieldBehavior; using ::cel::BindProtoToActivation; +using ::cel::BindProtoViewToActivation; namespace protobuf_internal { diff --git a/runtime/BUILD b/runtime/BUILD index da2192439..6831aeaae 100644 --- a/runtime/BUILD +++ b/runtime/BUILD @@ -116,10 +116,10 @@ cc_library( hdrs = ["bind_proto_to_activation.h"], deps = [ ":activation", - "//common:casting", "//common:value", "//internal:status_macros", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/runtime/bind_proto_to_activation.h b/runtime/bind_proto_to_activation.h index 9bc866937..03476b386 100644 --- a/runtime/bind_proto_to_activation.h +++ b/runtime/bind_proto_to_activation.h @@ -18,9 +18,9 @@ #include #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" -#include "common/casting.h" #include "common/value.h" #include "runtime/activation.h" #include "google/protobuf/arena.h" @@ -49,6 +49,43 @@ absl::Status BindProtoToActivation( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation); +template +absl::Status BindProtoToActivationImpl( + const T& context, BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + static_assert(std::is_base_of_v); + + Value parent; + if constexpr (kBorrow) { + parent = Value::WrapMessageUnsafe(&context, descriptor_pool, + message_factory, arena); + } else { + parent = + Value::FromMessage(context, descriptor_pool, message_factory, arena); + } + + if (!parent.IsStruct()) { + return absl::InvalidArgumentError( + absl::StrCat("context is a well-known type: ", context.GetTypeName())); + } + StructValue struct_value = parent.GetStruct(); + + const google::protobuf::Descriptor* descriptor = context.GetDescriptor(); + ABSL_DCHECK(descriptor != nullptr); + if (descriptor == nullptr) { + // Generally not possible, but don't crash in case of a misbehaving + // implementation in normal builds. + return absl::InvalidArgumentError( + absl::StrCat("context missing descriptor: ", context.GetTypeName())); + } + + return BindProtoToActivation(*descriptor, struct_value, unset_field_behavior, + descriptor_pool, message_factory, arena, + activation); +} + } // namespace runtime_internal // Utility method, that takes a protobuf Message and interprets it as a @@ -89,26 +126,9 @@ absl::Status BindProtoToActivation( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { - static_assert(std::is_base_of_v); - Value parent = - Value::FromMessage(context, descriptor_pool, message_factory, arena); - - if (!InstanceOf(parent)) { - return absl::InvalidArgumentError( - absl::StrCat("context is a well-known type: ", context.GetTypeName())); - } - const StructValue& struct_value = Cast(parent); - - const google::protobuf::Descriptor* descriptor = context.GetDescriptor(); - - if (descriptor == nullptr) { - return absl::InvalidArgumentError( - absl::StrCat("context missing descriptor: ", context.GetTypeName())); - } - - return runtime_internal::BindProtoToActivation( - *descriptor, struct_value, unset_field_behavior, descriptor_pool, - message_factory, arena, activation); + return runtime_internal::BindProtoToActivationImpl( + context, unset_field_behavior, descriptor_pool, message_factory, arena, + activation); } template @@ -122,6 +142,33 @@ absl::Status BindProtoToActivation( activation); } +// Like `BindProtoToActivation`, but uses `Value::WrapMessageUnsafe` to borrow +// from `context` rather than copying fields to `arena`. +// +// Requires the caller to keep the context message valid as long as the +// activation or any derived value. +template +absl::Status BindProtoViewToActivation( + const T& context, BindProtoUnsetFieldBehavior unset_field_behavior, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + return runtime_internal::BindProtoToActivationImpl( + context, unset_field_behavior, descriptor_pool, message_factory, arena, + activation); +} + +template +absl::Status BindProtoViewToActivation( + const T& context, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Activation* absl_nonnull activation) { + return BindProtoViewToActivation(context, BindProtoUnsetFieldBehavior::kSkip, + descriptor_pool, message_factory, arena, + activation); +} + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_RUNTIME_BIND_PROTO_TO_ACTIVATION_H_ diff --git a/runtime/bind_proto_to_activation_test.cc b/runtime/bind_proto_to_activation_test.cc index 2242f47c0..e7935b92a 100644 --- a/runtime/bind_proto_to_activation_test.cc +++ b/runtime/bind_proto_to_activation_test.cc @@ -240,5 +240,39 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationMapComplex) { IsOkAndHolds(Optional(IsMapValueOfSize(2)))); } +TEST_F(BindProtoToActivationTest, BindProtoViewToActivation) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT( + BindProtoViewToActivation(test_all_types, descriptor_pool(), + message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_int64", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(123)))); +} + +TEST_F(BindProtoToActivationTest, BindProtoViewToActivationDefault) { + TestAllTypes test_all_types; + test_all_types.set_single_int64(123); + Activation activation; + + ASSERT_THAT( + BindProtoViewToActivation( + test_all_types, BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool(), message_factory(), arena(), &activation), + IsOk()); + + EXPECT_THAT(activation.FindVariable("single_int32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(-32)))); + EXPECT_THAT(activation.FindVariable("single_sint32", descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Optional(IntValueIs(0)))); +} + } // namespace } // namespace cel From bc6a2e2abc6c797f3e90bd298fa8ad6e363793b5 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 10 Jul 2026 15:43:50 -0700 Subject: [PATCH 117/350] Simplify program layout for variadic and/or. PiperOrigin-RevId: 945926874 --- eval/compiler/flat_expr_builder.cc | 20 +++++++++++++------- eval/eval/BUILD | 1 - eval/eval/jump_step.cc | 7 ++++--- eval/eval/jump_step.h | 3 ++- eval/eval/logic_step.cc | 10 ++++++---- eval/eval/logic_step.h | 6 ++++-- eval/eval/logic_step_test.cc | 4 +++- 7 files changed, 32 insertions(+), 19 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index fda57ca73..fc7289bd2 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -2172,21 +2172,27 @@ void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { return; } const int last_arg_index = expr->call_expr().args().size() - 1; - if (arg_num > 0) { + const size_t num_args = expr->call_expr().args().size(); + if (arg_num == last_arg_index) { if (is_or_) { - visitor_->AddStep(CreateOrStep(expr->id())); + visitor_->AddStep(CreateOrStep(num_args, expr->id())); } else { - visitor_->AddStep(CreateAndStep(expr->id())); + visitor_->AddStep(CreateAndStep(num_args, expr->id())); } if (short_circuiting_ && !jump_steps_.empty()) { - visitor_->SetProgressStatusIfError( - jump_steps_.back().set_target(visitor_->GetCurrentIndex())); + for (auto& jump : jump_steps_) { + visitor_->SetProgressStatusIfError( + jump.set_target(visitor_->GetCurrentIndex())); + } } } if (short_circuiting_ && arg_num < last_arg_index) { std::unique_ptr jump_step = - is_or_ ? CreateCondJumpStep(true, {}, expr->id()) - : CreateCondJumpStep(false, {}, expr->id()); + is_or_ + ? CreateCondJumpStep(true, {}, /*expected_stack_size=*/arg_num + 1, + expr->id()) + : CreateCondJumpStep(false, {}, /*expected_stack_size=*/arg_num + 1, + expr->id()); ProgramStepIndex index = visitor_->GetCurrentIndex(); if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); jump_step_ptr) { diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 44c7ded79..c0f544405 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -418,7 +418,6 @@ cc_library( "//common:value", "//eval/internal:errors", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", ], diff --git a/eval/eval/jump_step.cc b/eval/eval/jump_step.cc index 60c8c039e..243a02e8a 100644 --- a/eval/eval/jump_step.cc +++ b/eval/eval/jump_step.cc @@ -143,9 +143,10 @@ class BoolCheckJumpStep : public JumpStepBase { // Factory method for Conditional Jump step. std::unique_ptr CreateCondJumpStep( - bool jump_condition, absl::optional jump_offset, int64_t expr_id) { - return std::make_unique(jump_condition, jump_offset, 1, - expr_id); + bool jump_condition, absl::optional jump_offset, + size_t expected_stack_size, int64_t expr_id) { + return std::make_unique(jump_condition, jump_offset, + expected_stack_size, expr_id); } // Factory method for Ternary Conditional Jump step. diff --git a/eval/eval/jump_step.h b/eval/eval/jump_step.h index 27deba352..d8555ae10 100644 --- a/eval/eval/jump_step.h +++ b/eval/eval/jump_step.h @@ -53,7 +53,8 @@ std::unique_ptr CreateJumpStep(absl::optional jump_offset, // It is compared to jump_condition, and if matched, jump is performed. // The boolean value is left on top of the stack. std::unique_ptr CreateCondJumpStep( - bool jump_condition, absl::optional jump_offset, int64_t expr_id); + bool jump_condition, absl::optional jump_offset, + size_t expected_stack_size, int64_t expr_id); // Factory method for Ternary Conditional Jump step. // Requires a boolean condition value on top of the stack. diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index 6d5c8bbb3..9e65db8f0 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -450,13 +450,15 @@ std::unique_ptr CreateDirectOrStep( } // Factory method for "And" Execution step -absl::StatusOr> CreateAndStep(int64_t expr_id) { - return std::make_unique(OpType::kAnd, 2, expr_id); +absl::StatusOr> CreateAndStep(size_t num_args, + int64_t expr_id) { + return std::make_unique(OpType::kAnd, num_args, expr_id); } // Factory method for "Or" Execution step -absl::StatusOr> CreateOrStep(int64_t expr_id) { - return std::make_unique(OpType::kOr, 2, expr_id); +absl::StatusOr> CreateOrStep(size_t num_args, + int64_t expr_id) { + return std::make_unique(OpType::kOr, num_args, expr_id); } // Factory method for recursive logical not "!" Execution step diff --git a/eval/eval/logic_step.h b/eval/eval/logic_step.h index d75ed3715..4f5be2615 100644 --- a/eval/eval/logic_step.h +++ b/eval/eval/logic_step.h @@ -23,10 +23,12 @@ std::unique_ptr CreateDirectOrStep( bool shortcircuiting); // Factory method for "And" Execution step -absl::StatusOr> CreateAndStep(int64_t expr_id); +absl::StatusOr> CreateAndStep(size_t num_args, + int64_t expr_id); // Factory method for "Or" Execution step -absl::StatusOr> CreateOrStep(int64_t expr_id); +absl::StatusOr> CreateOrStep(size_t num_args, + int64_t expr_id); // Factory method for recursive logical not "!" Execution step std::unique_ptr CreateDirectNotStep( diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index 17ca8ba0d..2bdcbc8ea 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -73,7 +73,9 @@ class LogicStepTest : public testing::TestWithParam { CEL_ASSIGN_OR_RETURN(step, CreateIdentStep("name1", /*expr_id=*/-1)); path.push_back(std::move(step)); - CEL_ASSIGN_OR_RETURN(step, (is_or) ? CreateOrStep(2) : CreateAndStep(2)); + CEL_ASSIGN_OR_RETURN( + step, (is_or) ? CreateOrStep(/*num_args=*/2, /*expr_id=*/2) + : CreateAndStep(/*num_args=*/2, /*expr_id=*/2)); path.push_back(std::move(step)); auto dummy_expr = std::make_unique(); From e5b3805cad9d683b808f5d90ae17574727eadb1c Mon Sep 17 00:00:00 2001 From: Clayton Knittel Date: Fri, 10 Jul 2026 16:22:38 -0700 Subject: [PATCH 118/350] Stamp out destructor calls to message objects in `cel::Unique`. PiperOrigin-RevId: 945944423 --- common/memory.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/common/memory.h b/common/memory.h index b19f54f94..615bce519 100644 --- a/common/memory.h +++ b/common/memory.h @@ -599,6 +599,11 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Unique final { friend struct std::pointer_traits>; friend struct ArenaTraits>; + static constexpr bool kNeedsArenaDestructor = + !std::is_trivially_destructible_v && + !google::protobuf::Arena::is_destructor_skippable::value && + !std::is_base_of_v; + Unique(T* ptr, uintptr_t arena) noexcept : ptr_(ptr), arena_(arena) {} Unique(T* ptr, google::protobuf::Arena* arena, bool unowned = false) noexcept @@ -619,8 +624,7 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Unique final { if ((arena_ & common_internal::kUniqueArenaBits) == common_internal::kUniqueArenaUnownedBit) { // We never registered the destructor, call it if necessary. - if constexpr (!std::is_trivially_destructible_v && - !google::protobuf::Arena::is_destructor_skippable::value) { + if constexpr (kNeedsArenaDestructor) { std::destroy_at(ptr_); } } @@ -631,8 +635,7 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Unique final { } void PreRelease() noexcept { - if constexpr (!std::is_trivially_destructible_v && - !google::protobuf::Arena::is_destructor_skippable::value) { + if constexpr (kNeedsArenaDestructor) { if (static_cast(*this) && (arena_ & common_internal::kUniqueArenaBits) == common_internal::kUniqueArenaUnownedBit) { From af2c3601fae1c5f8e81806b9cdd0461b74e28dec Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 14 Jul 2026 13:01:49 -0700 Subject: [PATCH 119/350] Update protobuf dependency version to 35.1. Updates the protobuf version in the Bazel MODULE file from 34.1 to 35.1. PiperOrigin-RevId: 947857690 --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 0901db5f8..090309f2a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -34,7 +34,7 @@ bazel_dep( bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep( name = "protobuf", - version = "34.1", + version = "35.1", repo_name = "com_google_protobuf", ) bazel_dep( From 8818be6907115a152f2d1048fcc27ca90e6bf1b9 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 15 Jul 2026 11:20:51 -0700 Subject: [PATCH 120/350] Update codelab to use the cel::Value/cel::Runtime APIs The modern APIs should be preferred for new usages going forward. A later update will provide more guidance on how/when to migrate from google::api::expr::runtime APIs. PiperOrigin-RevId: 948445880 --- codelab/BUILD | 96 ++--- codelab/README.md | 382 ++++++++++-------- codelab/cel_compiler.h | 47 --- codelab/cel_compiler_test.cc | 146 ------- codelab/checked_expr_conversion_example.h | 59 +++ .../checked_expr_conversion_example_test.cc | 60 +++ codelab/exercise1.cc | 85 ++-- codelab/exercise10.cc | 5 +- codelab/exercise2.cc | 122 +++--- codelab/exercise2.h | 6 +- codelab/exercise4.cc | 121 +++--- codelab/solutions/BUILD | 62 +-- codelab/solutions/exercise1.cc | 116 +++--- codelab/solutions/exercise10.cc | 5 +- codelab/solutions/exercise2.cc | 119 +++--- codelab/solutions/exercise4.cc | 140 ++++--- 16 files changed, 797 insertions(+), 774 deletions(-) delete mode 100644 codelab/cel_compiler.h delete mode 100644 codelab/cel_compiler_test.cc create mode 100644 codelab/checked_expr_conversion_example.h create mode 100644 codelab/checked_expr_conversion_example_test.cc diff --git a/codelab/BUILD b/codelab/BUILD index 69c2825e2..f61b586eb 100644 --- a/codelab/BUILD +++ b/codelab/BUILD @@ -44,18 +44,22 @@ cc_library( "nobuilder", ], deps = [ - "//eval/public:activation", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", + "//checker:validation_result", + "//common:ast", + "//common:minimal_descriptor_pool", + "//common:value", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//internal:status_macros", - "//parser", + "//runtime", + "//runtime:activation", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -76,22 +80,25 @@ cc_library( srcs = ["exercise2.cc"], hdrs = ["exercise2.h"], deps = [ - ":cel_compiler", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:type", + "//common:value", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", "//internal:status_macros", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", ], @@ -125,42 +132,30 @@ cc_test( ) cc_library( - name = "cel_compiler", - hdrs = ["cel_compiler.h"], + name = "checked_expr_conversion_example", + hdrs = ["checked_expr_conversion_example.h"], deps = [ - "//checker:validation_result", + "//common:ast", "//common:ast_proto", - "//compiler", "//internal:status_macros", - "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", ], ) cc_test( - name = "cel_compiler_test", - srcs = ["cel_compiler_test.cc"], + name = "checked_expr_conversion_example_test", + srcs = ["checked_expr_conversion_example_test.cc"], deps = [ - ":cel_compiler", - "//common:decl", - "//common:type", + ":checked_expr_conversion_example", + "//checker:validation_result", + "//common:ast", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:activation_bind_helper", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_function_adapter", - "//eval/public:cel_value", - "//eval/public/testing:matchers", "//internal:testing", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", + "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -170,22 +165,27 @@ cc_library( srcs = ["exercise4.cc"], hdrs = ["exercise4.h"], deps = [ - ":cel_compiler", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", + "//common:decl", + "//common:type", + "//common:value", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:activation_bind_helper", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", "//internal:status_macros", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:function_adapter", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", + "@com_google_absl//absl/types:optional", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", ], @@ -270,6 +270,7 @@ cc_library( deps = [ ":network_functions", "//checker:validation_result", + "//common:ast", "//common:decl", "//common:minimal_descriptor_pool", "//common:type", @@ -277,6 +278,7 @@ cc_library( "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", + "//internal:status_macros", "//runtime", "//runtime:activation", "//runtime:runtime_builder", diff --git a/codelab/README.md b/codelab/README.md index 96f7598ba..b20a5db31 100644 --- a/codelab/README.md +++ b/codelab/README.md @@ -1,42 +1,59 @@ # What is CEL? -Common Expression Language (CEL) is an expression language that’s fast, portable, and safe to execute in performance-critical applications. CEL is designed to be embedded in an application, with application-specific extensions, and is ideal for extending declarative configurations that your applications might already use. + +Common Expression Language (CEL) is an expression language that’s fast, +portable, and safe to execute in performance-critical applications. CEL is +designed to be embedded in an application, with application-specific extensions, +and is ideal for extending declarative configurations that your applications +might already use. ## What is covered in this Codelab? -This codelab is aimed at developers who would like to learn CEL to use services that already support CEL. This Codelab covers common use cases. This codelab doesn't cover how to integrate CEL into your own project. For a more in-depth look at the language, semantics, and features see the [CEL Language Definition on GitHub](https://github.com/google/cel-spec). + +This codelab is aimed at developers who would like to learn CEL to use services +that already support CEL. This Codelab covers common use cases. This codelab +doesn't cover how to integrate CEL into your own project. For a more in-depth +look at the language, semantics, and features see the +[CEL Language Definition on GitHub](https://github.com/google/cel-spec). Some key areas covered are: -* [Hello, World: Using CEL to evaluate a String](#hello-world) -* [Creating variables](#creating-variables) -* [Commutative logical AND/OR](#logical-andor) -* [Adding custom functions](#custom-functions) +* [Hello, World: Using CEL to evaluate a String](#hello-world) +* [Creating variables](#creating-variables) +* [Commutative logical AND/OR](#logical-andor) +* [Adding custom functions](#custom-functions) ### Prerequisites -This codelab builds upon a basic understanding of Protocol Buffers and C++. -If you're not familiar with Protocol Buffers, the first exercise will give you a sense of how CEL works, but because the more advanced examples use Protocol Buffers as the input into CEL, they may be harder to understand. Consider working through one of these tutorials, first. See the devsite for [Protocol Buffers](https://protobuf.dev). +This codelab builds upon a basic understanding of Protocol Buffers and C++. -Notes on portability: Protocol Buffers are not required to use CEL -generally, but the C++ implementation has a hard dependency on the library -and some APIs reference protobuf types directly. Automated builds test -against gcc9 and clang11 on linux. We accept requests for portability -fixes for other OSes and compilers, but don't actively maintain support at -this time. A simple Docker file is provided as a reference for a known good -environment configuration for running the codelab solutions. +If you're not familiar with Protocol Buffers, the first exercise will give you a +sense of how CEL works, but because the more advanced examples use Protocol +Buffers as the input into CEL, they may be harder to understand. Consider +working through one of these tutorials, first. See the devsite for +[Protocol Buffers](https://protobuf.dev). + +Notes on portability: Protocol Buffers are not required to use CEL generally, +but the C++ implementation has a hard dependency on the library and some APIs +reference protobuf types directly. Automated builds test against gcc10 and +clang14 on linux, a recent visual code version on Windows, and a recent xcode +version on MacOS. We accept requests for portability fixes for other OSes and +compilers, but don't actively maintain support at this time. A simple Docker +file is provided as a reference for a known good environment configuration for +running the codelab solutions. What you'll need: -- Git -- Bazel -- C/C++ Compiler (GCC, Clang, Visual Studio). -- Optional: bazelisk is a wrapper around bazel that simplifies version - management. If using, substitute all bazel commands below with `bazelisk`. +- Git +- Bazel +- C/C++ Compiler (GCC, Clang, Visual Studio). +- Optional: bazelisk is a wrapper around bazel that simplifies version + management. If using, substitute all bazel commands below with `bazelisk`. ## GitHub Setup GitHub Repo: -The code for this codelab lives in the `codelab` folder of the cel-cpp repo. The solution is available in the `codelab/solution` folder of the same repo. +The code for this codelab lives in the `codelab` folder of the cel-cpp repo. The +solutions are available in the `codelab/solution` folder of the same repo. Clone and cd into the repo: @@ -52,68 +69,74 @@ bazel build //codelab:all ``` ## Hello, World -In the tried and true tradition of all programming languages, let's start with "Hello, World!". + +In the tried and true tradition of all programming languages, let's start with +"Hello, World!". Update exercise1.cc with the following: Using declarations: ```c++ -using ::google::api::expr::parser::Parse; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelExpression; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; +using ::cel::Activation; +using ::cel::Compiler; +using ::cel::CompilerBuilder; +using ::cel::Program; +using ::cel::Runtime; +using ::cel::RuntimeBuilder; +using ::cel::RuntimeOptions; +using ::cel::ValidationResult; +using ::cel::Value; ``` Implementation: ```c++ -absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr) -{ +absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr) { // === Start Codelab === - // Setup a default environment for building expressions. - InterpreterOptions options; - std::unique_ptr builder = - CreateCelExpressionBuilder(options); - + // Setup a default compiler for compiling expressions. + CEL_ASSIGN_OR_RETURN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::GetMinimalDescriptorPool())); CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); - - // Parse the expression. This is fine for codelabs, but this skips the type - // checking phase. It won't check that functions and variables are available - // in the environment, and it won't handle certain ambiguous identifier - // expressions (e.g. container lookup vs namespaced name, packaged function - // vs. receiver call style function). - ParsedExpr parsed_expr; - CEL_ASSIGN_OR_RETURN(parsed_expr, Parse(cel_expr)); - - // The evaluator uses a proto Arena for incidental allocations during - // evaluation. + compiler_builder->AddLibrary(cel::StandardCompilerLibrary())); + CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + + // Compile the expression. + CEL_ASSIGN_OR_RETURN(ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } + + // Setup a standard runtime for evaluating expressions. + RuntimeOptions options; + CEL_ASSIGN_OR_RETURN( + RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), + options)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + // Build the executable program from the compiled AST. + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + // The evaluator uses a proto Arena for allocations during evaluation. proto2::Arena arena; - // The activation provides variables and functions that are bound into the + // The activation provides variables and functions bound into the // expression environment. In this example, there's no context expected, so - // we just provide an empty one to the evaluator. + // we provide an empty activation. Activation activation; - // Build the expression plan. This assumes that the source expression AST and - // the expression builder outlives the CelExpression object. - CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - - // Actually run the expression plan. We don't support any environment - // variables at the moment so just use an empty activation. - CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, &arena)); - - // Convert the result to a c++ string. CelValues may reference instances from - // either the input expression, or objects allocated on the arena, so we need - // to pass ownership (in this case by copying to a new instance and returning - // that). + // Run the program. + CEL_ASSIGN_OR_RETURN(Value result, + program->Evaluate(&arena, activation)); + + // Convert the result to a C++ string. return ConvertResult(result); // === End Codelab === } @@ -129,151 +152,178 @@ You can add additional test cases or experiment with different return types. Hello, World! Now, let's break down what's happening. - ### Setup the Environment -CEL applications evaluate an expression against an environment. -The standard CEL environment supports all of the types, operators, functions, and macros defined within the language spec. The environment can be customized by providing options to disable macros, declare custom variables and functions, etc. +CEL applications compile and evaluate an expression against an environment. + +The standard CEL environment supports all of the types, operators, functions, +and macros defined within the language spec. The compiler and runtime objects +can be customized with other variables, types, and functions. -An ExpressionBuilder maintains C++ evaluation environment. This creates a builder with the standard environment. +A `CompilerBuilder` configures the compilation environment, while a +`RuntimeBuilder` configures the execution environment: ```c++ -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_options.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" ... -// Setup a default environment for building expressions. - -// Breaking behavior changes and optional features are controlled by -// InterpreterOptions. -InterpreterOptions options; - -// Environment used for planning and evaluating expressions is managed by an -// ExpressionBuilder. -std::unique_ptr builder = - CreateCelExpressionBuilder(options); - -// Add standard function bindings e.g. for +,-,==,||,&& operators. -// Custom functions (implementing the CelFunction interface) can be added to the -// registry similarly. +// Create a compiler configured with the standard CEL library. +CEL_ASSIGN_OR_RETURN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::GetMinimalDescriptorPool())); CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); + compiler_builder->AddLibrary(cel::StandardCompilerLibrary())); +CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); + +// Create a standard runtime. +cel::RuntimeOptions options; +CEL_ASSIGN_OR_RETURN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), + options)); +CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); ``` -### Parse -After the environment is configured, you can parse and check the expressions: +### Compile + +After the compiler is configured, you can compile source expressions into a +checked `cel::Ast`: ```c++ -#include "google/api/expr/syntax.proto.h" -#include "parser/parser.h" +#include "checker/validation_result.h" // ... -ASSIGN_OR_RETURN(google::api::expr::ParsedExpr parsed_expr, google::api::expr::parser::Parse(cel_expr)); +CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); +if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); +} ``` -The C++ parser is a stand-alone utility. It's not aware of the evaluation environment and does not perform any semantic checks on the expression. A status is returned if the input string isn't a syntactically valid CEL expression or if it exceeds the configured complexity limits (see cel::ParserOptions and default limits). +`compiler->Compile(...)` parses and type-checks the expression in one step, +returning a `cel::ValidationResult`. If the expression is valid +(`validation_result.IsValid()`), the resulting `cel::Ast` can be extracted using +`validation_result.ReleaseAst()`. + +> Note: If you need to interface with external systems that store or transmit +> serialized protobuf ASTs (`google::api::expr::CheckedExpr`), conversion +> examples are available in `checked_expr_conversion_example.h` +> (`cel::AstToCheckedExpr` and `cel::CreateAstFromCheckedExpr`). ### Evaluate -After the expressions have been parsed and checked into an AST representation, it can be converted into an evaluable program whose function bindings and evaluation modes can be customized depending on the stack you are using. -Once a CEL expression is planned, it can be evaluated against an evaluation context (an activation). The evaluation result will be either a value or an error state. -The InterpreterOptions to create the expression plan are honored at evaluation. C++ uses the proto representation of either a parsed `google.api.expr.ParsedExpr` or parsed and type-checked `google.api.expr.CheckedExpr` AST directly. -Once a CEL program is planned (represented by a `google::api::expr::runtime::CelExpression`), it can be evaluated against an `google::api::expr::runtime::Activation`. The Activation provides per-evaluation bindings for variables and functions in the expression's environment. + +Once a `cel::Ast` is compiled, `runtime->CreateProgram(...)` creates an +executable `cel::Program`. The program is then evaluated against a +`cel::Activation`, which provides per-evaluation bindings for variables and +functions in the environment: ```c++ +#include "common/ast.h" +#include "runtime/activation.h" #include "third_party/protobuf/arena.h" -#include "eval/public/activation.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_value.h" -#include "internal/status_macros.h" -#include "parser/parser.h" ... -// The evaluator uses a proto Arena for incidental allocations during -// evaluation. +// Build the executable program from the compiled AST. +CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); +CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + +// Run the program against an activation and proto Arena. +cel::Activation activation; proto2::Arena arena; -// The activation provides variables and functions that are bound into the -// expression environment. In this example, there's no context expected, so -// we just provide an empty one to the evaluator. -Activation activation; - -// Build the expression plan. This assumes that the source expression AST and -// the expression builder outlives the CelExpression object. -CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - -// Actually run the expression plan. We don't support any environment -// variables at the moment so just use an empty activation. -CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, &arena)); - -// Convert the result to a C++ string. CelValues may reference instances from -// either the input expression, or objects allocated on the arena, so we need -// to pass ownership (in this case by copying to a new instance and returning -// that). +CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena, activation)); + return ConvertResult(result); ``` ## Creating variables -Most CEL applications will declare variables that can be referenced within expressions. Variables declarations specify a name and a type. A variable's type may either be a CEL builtin type, a protocol buffer well-known type, or any protobuf message type so long as its descriptor is also provided to CEL. -At runtime, the hosting program binds instances of variables to the evaluation context (using the variable name as a key). +Most CEL applications declare variables that can be referenced within +expressions. Variable declarations specify a name and a type. A variable's type +may be a CEL built-in type, a protocol buffer well-known type, or any protobuf +message type so long as its descriptor is registered. + +At runtime, the hosting program binds variable values into `cel::Activation`. -For the C++ evaluator at runtime, the values are managed by the `google::api::expr::runtime::CelValue` type, a variant over the C++ representations of supported CEL types. +For the C++ evaluator at runtime, values are represented by `cel::Value`. Update exercise2.cc: ```c++ -// The Variables exercise shows how to declare and use variables in expressions. -// There are two overloads for preparing an expression either granularly for -// individual variables or using a helper to bind a context proto. - -// The first overload shows manually populating individual variables in the -// evaluation environment. This allows cel_expr to reference 'bool_var'. -absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr, - bool bool_var) { - Activation activation; +absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, + bool bool_var) { + CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + MakeCelCompiler()); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } + + cel::Activation activation; proto2::Arena arena; // === Start Codelab === - activation.InsertValue("bool_var", CelValue::CreateBool(bool_var)); - // === End Codelab === - - return ParseAndEvaluate(cel_expr, activation, &arena); + activation.InsertOrAssignValue("bool_var", cel::BoolValue(bool_var)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } ``` -Run the following to check your work. You should have fixed the first two test cases in exercise2_test.cc. +Run the following to check your work. You should have fixed the first two test +cases in exercise2_test.cc. ``` bazel test //codelab:exercise2_test ``` -The second overload uses a protocol buffer message to represent the environment variables. For this use case, there is a helper to automatically bind in fields from a top level message (see `google::api::expr::runtime::BindProtoToActivation`). In this example, we assume that unset fields should be bound to default values. +The second overload uses a protocol buffer message (`AttributeContext`) to +represent the environment variables. For this use case, +`cel::BindProtoToActivation` automatically binds fields from a top-level message +into the activation (`#include +"runtime/bind_proto_to_activation.h"`): ```c++ -#include "eval/public/activation_bind_helper.h" +#include "runtime/bind_proto_to_activation.h" // ... -using ::google::api::expr::runtime::ProtoUnsetFieldOptions; -// ... -absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr, - const AttributeContext& context) { - Activation activation; - google::protobuf::Arena arena; +absl::StatusOr CompileAndEvaluateWithContext( + absl::string_view cel_expr, const AttributeContext& context) { + // ... compile cel_expr ... + cel::Activation activation; + proto2::Arena arena; // === Start Codelab === - - CEL_RETURN_IF_ERROR(BindProtoToActivation( - &context, &arena, &activation, ProtoUnsetFieldOptions::kBindDefault)); + CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + proto2::DescriptorPool::generated_pool(), + proto2::MessageFactory::generated_factory(), &arena, &activation)); // === End Codelab === - return ParseAndEvaluate(cel_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } ``` -Note: You can experiment with unset values and the alternative bind option for BindProtoToActivation. With ProtoUnsetFieldOptions::kSkip unset values will not be bound at all, and accesses in expressions will cause errors. +Note: You can experiment with unset values using +`cel::BindProtoUnsetFieldBehavior::kBindDefaultValue` or `kSkip`. With `kSkip`, +unset protobuf fields will not be bound at all, and accesses in expressions will +result in errors. ## Logical And/Or -One of CEL's more distinctive features is its use of commutative logical operators. Either side of a conditional branch can short-circuit the evaluation, even in the face of errors or partial input. -Note: If you are skipping ahead, copy the solution for exercise2 -- we'll be using it to test the behavior of some simple expressions. -exercise3_test.cc lists truth tables for simple expressions using the 'or', 'and', and 'ternary' operators. +One of CEL's more distinctive features is its use of commutative logical +operators. Either side of a conditional branch can short-circuit the evaluation, +even in the face of errors or partial input. Note: If you are skipping ahead, +copy the solution for exercise2 -- we'll be using it to test the behavior of +some simple expressions. + +exercise3_test.cc lists truth tables for simple expressions using the 'or', +'and', and 'ternary' operators. Running the following should result in some failing expectations. @@ -309,7 +359,8 @@ TEST(Exercise3Var, LogicalOr) { } ``` -Updating the two failing cases "true || (1 / 0 > 2)" and "(1 / 0 > 2) || true" should fix this test: +Updating the two failing cases "true || (1 / 0 > 2)" and "(1 / 0 > 2) || true" +should fix this test: ```c++ // ... @@ -323,6 +374,15 @@ Updating the two failing cases "true || (1 / 0 > 2)" and "(1 / 0 > 2) || true" s IsOkAndHolds(true)); ``` -You can examine the other tests for other cases for corresponding behavior for the 'and' and ternary operators. +You can examine the other tests for other cases for corresponding behavior for +the 'and' and ternary operators. + +CEL finds an evaluation order which gives results whenever possible, ignoring +errors or even missing data that might occur in other evaluation orders. +Applications like IAM conditions rely on this property to minimize the cost of +evaluation, deferring the gathering of expensive inputs when a result can be +reached without them. + +## Adding custom functions {#custom-functions} -CEL finds an evaluation order which gives results whenever possible, ignoring errors or even missing data that might occur in other evaluation orders. Applications like IAM conditions rely on this property to minimize the cost of evaluation, deferring the gathering of expensive inputs when a result can be reached without them. +See `exercise4.h` and `exercise4.cc`. Documentation to be added later. diff --git a/codelab/cel_compiler.h b/codelab/cel_compiler.h deleted file mode 100644 index 0ff2f699b..000000000 --- a/codelab/cel_compiler.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_CODELAB_COMPILER_H_ -#define THIRD_PARTY_CEL_CPP_CODELAB_COMPILER_H_ - -#include "cel/expr/checked.pb.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/string_view.h" -#include "checker/validation_result.h" -#include "common/ast_proto.h" -#include "compiler/compiler.h" -#include "internal/status_macros.h" - -namespace cel_codelab { - -// Helper for compiling expression and converting to proto. -// -// Simplifies error handling for brevity in the codelab. -inline absl::StatusOr CompileToCheckedExpr( - const cel::Compiler& compiler, absl::string_view expr) { - CEL_ASSIGN_OR_RETURN(cel::ValidationResult result, compiler.Compile(expr)); - - if (!result.IsValid() || result.GetAst() == nullptr) { - return absl::InvalidArgumentError(result.FormatError()); - } - - cel::expr::CheckedExpr pb; - CEL_RETURN_IF_ERROR(cel::AstToCheckedExpr(*result.GetAst(), &pb)); - return pb; -}; - -} // namespace cel_codelab - -#endif // THIRD_PARTY_CEL_CPP_CODELAB_COMPILER_H_ diff --git a/codelab/cel_compiler_test.cc b/codelab/cel_compiler_test.cc deleted file mode 100644 index 635b4d54d..000000000 --- a/codelab/cel_compiler_test.cc +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "codelab/cel_compiler.h" - -#include -#include - -#include "google/rpc/context/attribute_context.pb.h" -#include "absl/log/absl_check.h" -#include "absl/status/status.h" -#include "absl/status/status_matchers.h" -#include "common/decl.h" -#include "common/type.h" -#include "compiler/compiler.h" -#include "compiler/compiler_factory.h" -#include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/activation_bind_helper.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_function_adapter.h" -#include "eval/public/cel_value.h" -#include "eval/public/testing/matchers.h" -#include "internal/testing.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/message.h" - -namespace cel_codelab { -namespace { - -using ::absl_testing::IsOk; -using ::absl_testing::StatusIs; -using ::cel::BoolType; -using ::cel::MakeFunctionDecl; -using ::cel::MakeOverloadDecl; -using ::cel::MakeVariableDecl; -using ::cel::StringType; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::BindProtoToActivation; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::FunctionAdapter; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; -using ::google::api::expr::runtime::test::IsCelBool; -using ::google::rpc::context::AttributeContext; -using ::testing::HasSubstr; - -std::unique_ptr MakeDefaultCompilerBuilder() { - google::protobuf::LinkMessageReflection(); - auto builder = - cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool()); - ABSL_CHECK_OK(builder.status()); - - ABSL_CHECK_OK((*builder)->AddLibrary(cel::StandardCompilerLibrary())); - ABSL_CHECK_OK((*builder)->GetCheckerBuilder().AddContextDeclaration( - "google.rpc.context.AttributeContext")); - - return std::move(builder).value(); -} - -TEST(DefaultCompiler, Basic) { - ASSERT_OK_AND_ASSIGN(auto compiler, MakeDefaultCompilerBuilder()->Build()); - EXPECT_THAT(compiler->Compile("1 < 2").status(), IsOk()); -} - -TEST(DefaultCompiler, AddFunctionDecl) { - auto builder = MakeDefaultCompilerBuilder(); - ASSERT_OK_AND_ASSIGN( - cel::FunctionDecl decl, - MakeFunctionDecl("IpMatch", - MakeOverloadDecl("IpMatch_string_string", BoolType(), - StringType(), StringType()))); - EXPECT_THAT(builder->GetCheckerBuilder().AddFunction(decl), IsOk()); - - ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); - - EXPECT_THAT(CompileToCheckedExpr( - *compiler, "IpMatch('255.255.255.255', '255.255.255.255')") - .status(), - IsOk()); - EXPECT_THAT( - CompileToCheckedExpr(*compiler, "IpMatch('255.255.255.255', 123436)") - .status(), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("no matching overload"))); -} - -TEST(DefaultCompiler, EndToEnd) { - google::protobuf::Arena arena; - - auto compiler_builder = MakeDefaultCompilerBuilder(); - ASSERT_OK_AND_ASSIGN( - cel::FunctionDecl func_decl, - MakeFunctionDecl("MyFunc", MakeOverloadDecl("MyFunc", BoolType()))); - ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddFunction(func_decl), - IsOk()); - - ASSERT_THAT(compiler_builder->GetCheckerBuilder().AddVariable( - MakeVariableDecl("my_var", BoolType())), - IsOk()); - - ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); - - ASSERT_OK_AND_ASSIGN( - auto expr, - CompileToCheckedExpr( - *compiler, - "(my_var || MyFunc()) && request.host == 'www.google.com'")); - - auto builder = - CreateCelExpressionBuilder(google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); - ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry()), IsOk()); - ASSERT_THAT(FunctionAdapter::CreateAndRegister( - "MyFunc", false, [](google::protobuf::Arena*) { return true; }, - builder->GetRegistry()), - IsOk()); - - ASSERT_OK_AND_ASSIGN(auto plan, builder->CreateExpression(&expr)); - - AttributeContext context; - context.mutable_request()->set_host("www.google.com"); - Activation activation; - ASSERT_THAT(BindProtoToActivation(&context, &arena, &activation), IsOk()); - activation.InsertValue("my_var", CelValue::CreateBool(false)); - - ASSERT_OK_AND_ASSIGN(CelValue result, plan->Evaluate(activation, &arena)); - - EXPECT_THAT(result, IsCelBool(true)); -} - -} // namespace -} // namespace cel_codelab diff --git a/codelab/checked_expr_conversion_example.h b/codelab/checked_expr_conversion_example.h new file mode 100644 index 000000000..fc3783ddd --- /dev/null +++ b/codelab/checked_expr_conversion_example.h @@ -0,0 +1,59 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_CODELAB_CHECKED_EXPR_CONVERSION_EXAMPLE_H_ +#define THIRD_PARTY_CEL_CPP_CODELAB_CHECKED_EXPR_CONVERSION_EXAMPLE_H_ + +#include + +#include "cel/expr/checked.pb.h" +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/ast_proto.h" +#include "internal/status_macros.h" + +namespace cel_codelab { + +// Examples demonstrating how to convert between protobuf-based AST +// representations (cel::expr::CheckedExpr) and modern runtime AST +// representations (cel::Ast). +// +// When evaluating expressions with modern `cel::Runtime`, compilation via +// `cel::Compiler::Compile` returns a `cel::ValidationResult` holding a +// `cel::Ast` that can be passed directly to `runtime->CreateProgram(...)`. +// +// However, when working with external services or storage layers that persist +// or transmit serialized protobuf ASTs (`cel::expr::CheckedExpr`), +// conversion functions in `common/ast_proto.h` can be used +// to move between `cel::Ast` and `cel::expr::CheckedExpr`. + +// Example: Convert a runtime `cel::Ast` to a `cel::expr::CheckedExpr` +// proto. +inline absl::StatusOr ConvertAstToCheckedExpr( + const cel::Ast& ast) { + cel::expr::CheckedExpr checked_expr; + CEL_RETURN_IF_ERROR(cel::AstToCheckedExpr(ast, &checked_expr)); + return checked_expr; +} + +// Example: Convert a `cel::expr::CheckedExpr` proto to a runtime +// `cel::Ast`. +inline absl::StatusOr> ConvertCheckedExprToAst( + const cel::expr::CheckedExpr& checked_expr) { + return cel::CreateAstFromCheckedExpr(checked_expr); +} + +} // namespace cel_codelab + +#endif // THIRD_PARTY_CEL_CPP_CODELAB_CHECKED_EXPR_CONVERSION_EXAMPLE_H_ diff --git a/codelab/checked_expr_conversion_example_test.cc b/codelab/checked_expr_conversion_example_test.cc new file mode 100644 index 000000000..31fb41a55 --- /dev/null +++ b/codelab/checked_expr_conversion_example_test.cc @@ -0,0 +1,60 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "codelab/checked_expr_conversion_example.h" + +#include +#include + +#include "cel/expr/checked.pb.h" +#include "absl/status/status_matchers.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" +#include "internal/testing.h" +#include "google/protobuf/descriptor.h" + +namespace cel_codelab { +namespace { + +using ::absl_testing::IsOk; + +TEST(CheckedExprConversionExampleTest, ConvertAstToCheckedExprAndBack) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(cel::StandardCompilerLibrary()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr compiler, + std::move(builder)->Build()); + + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + compiler->Compile("1 + 2 == 3")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + validation_result.ReleaseAst()); + + // Convert cel::Ast to CheckedExpr proto + ASSERT_OK_AND_ASSIGN(cel::expr::CheckedExpr checked_expr, + ConvertAstToCheckedExpr(*ast)); + + // Convert CheckedExpr proto back to cel::Ast + ASSERT_OK_AND_ASSIGN(std::unique_ptr roundtrip_ast, + ConvertCheckedExprToAst(checked_expr)); + EXPECT_NE(roundtrip_ast, nullptr); +} + +} // namespace +} // namespace cel_codelab diff --git a/codelab/exercise1.cc b/codelab/exercise1.cc index de7ccf6e0..7ca81451b 100644 --- a/codelab/exercise1.cc +++ b/codelab/exercise1.cc @@ -14,69 +14,74 @@ #include "codelab/exercise1.h" -#include +#include // IWYU pragma: keep #include +#include // IWYU pragma: keep -#include "cel/expr/syntax.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "eval/public/activation.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" -#include "internal/status_macros.h" -#include "parser/parser.h" +#include "checker/validation_result.h" // IWYU pragma: keep, needed for codelab solution +#include "common/ast.h" // IWYU pragma: keep +#include "common/minimal_descriptor_pool.h" // IWYU pragma: keep +#include "common/value.h" +#include "compiler/compiler.h" // IWYU pragma: keep +#include "compiler/compiler_factory.h" // IWYU pragma: keep +#include "compiler/standard_library.h" // IWYU pragma: keep +#include "internal/status_macros.h" // IWYU pragma: keep +#include "runtime/activation.h" +#include "runtime/runtime.h" // IWYU pragma: keep +#include "runtime/runtime_builder.h" // IWYU pragma: keep +#include "runtime/runtime_options.h" // IWYU pragma: keep +#include "runtime/standard_runtime_builder_factory.h" // IWYU pragma: keep #include "google/protobuf/arena.h" namespace cel_codelab { namespace { -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelValue; - -// Convert the CelResult to a C++ string if it is string typed. Otherwise, -// return invalid argument error. This takes a copy to avoid lifecycle concerns -// (the evaluator may represent strings as stringviews backed by the input -// expression). -absl::StatusOr ConvertResult(const CelValue& value) { - if (CelValue::StringHolder inner_value; value.GetValue(&inner_value)) { - return std::string(inner_value.value()); - } else { - return absl::InvalidArgumentError(absl::StrCat( - "expected string result got '", CelValue::TypeName(value.type()), "'")); +// Convert the cel::Value to a C++ string if it is string typed. Otherwise, +// return invalid argument error. +absl::StatusOr ConvertResult(const cel::Value& value) { + if (value.IsString()) { + return value.GetString().ToString(); } + return absl::InvalidArgumentError( + absl::StrCat("expected string result got '", value.GetTypeName(), "'")); } + } // namespace absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr) { // === Start Codelab === - // Parse the expression using ::google::api::expr::parser::Parse; - // This will return a cel::expr::ParsedExpr message. + // 1. Setup a default compiler for compiling expressions: + // Use cel::NewCompilerBuilder(cel::GetMinimalDescriptorPool()) and add + // cel::StandardCompilerLibrary(). Build the cel::Compiler. + + // 2. Compile the expression using compiler->Compile(cel_expr). + // Check that the resulting validation_result.IsValid(). - // Setup a default environment for building expressions. - // std::unique_ptr builder = - // CreateCelExpressionBuilder(options); + // 3. Setup a standard runtime for evaluating expressions: + // Use cel::CreateStandardRuntimeBuilder(cel::GetMinimalDescriptorPool(), + // options) and build the cel::Runtime. - // Register standard functions. - // CEL_RETURN_IF_ERROR( - // RegisterBuiltinFunctions(builder->GetRegistry(), options)); + // 4. Create an executable program from the compiled AST: + // Use CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + // validation_result.ReleaseAst()) and + // runtime->CreateProgram(std::move(ast)). - // The evaluator uses a proto Arena for incidental allocations during - // evaluation. + // The evaluator uses a proto Arena for allocations during evaluation. google::protobuf::Arena arena; - // The activation provides variables and functions that are bound into the + // The activation provides variables and functions bound into the // expression environment. In this example, there's no context expected, so - // we just provide an empty one to the evaluator. - Activation activation; + // we provide an empty activation. + cel::Activation activation; + (void)arena; + (void)activation; - // Using the CelExpressionBuilder and the ParseExpr, create an execution plan - // (google::api::expr::runtime::CelExpression), evaluate, and return the - // result. Use the provided helper function ConvertResult to copy the value - // for return. + // 5. Evaluate the program and convert the result: + // Call program->Evaluate(&arena, activation) and pass the resulting + // cel::Value to ConvertResult. return absl::UnimplementedError("Not yet implemented"); // === End Codelab === } diff --git a/codelab/exercise10.cc b/codelab/exercise10.cc index 37eaa7642..bbe202cb3 100644 --- a/codelab/exercise10.cc +++ b/codelab/exercise10.cc @@ -23,6 +23,7 @@ #include "absl/strings/string_view.h" #include "checker/validation_result.h" #include "codelab/network_functions.h" +#include "common/ast.h" #include "common/decl.h" #include "common/minimal_descriptor_pool.h" #include "common/type.h" @@ -30,6 +31,7 @@ #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" +#include "internal/status_macros.h" #include "runtime/activation.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" @@ -95,8 +97,9 @@ absl::StatusOr CompileAndEvaluateExercise10(absl::string_view expression, return absl::InvalidArgumentError(checked->FormatError()); } + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, checked->ReleaseAst()); absl::StatusOr> program = - (*runtime)->CreateProgram(checked->ReleaseAst().value()); + (*runtime)->CreateProgram(std::move(ast)); if (!program.ok()) { return std::move(program).status(); diff --git a/codelab/exercise2.cc b/codelab/exercise2.cc index 373f63365..dd73dae62 100644 --- a/codelab/exercise2.cc +++ b/codelab/exercise2.cc @@ -15,25 +15,29 @@ #include "codelab/exercise2.h" #include +#include -#include "cel/expr/checked.pb.h" -#include "cel/expr/syntax.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "codelab/cel_compiler.h" +#include "checker/type_checker_builder.h" // IWYU pragma: keep, needed for codelab solution +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" // IWYU pragma: keep +#include "common/type.h" // IWYU pragma: keep +#include "common/value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" // IWYU pragma: keep +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -41,15 +45,6 @@ namespace cel_codelab { namespace { -using ::cel::expr::CheckedExpr; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelError; -using ::google::api::expr::runtime::CelExpression; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::google::rpc::context::AttributeContext; absl::StatusOr> MakeCelCompiler() { @@ -65,45 +60,41 @@ absl::StatusOr> MakeCelCompiler() { // === Start Codelab === // Add 'AttributeContext' as a context message to the type checker and a // boolean variable 'bool_var'. Relevant functions are on the - // TypeCheckerBuilder class (see CompilerBuilder::GetCheckerBuilder). + // cel::TypeCheckerBuilder class (see builder->GetCheckerBuilder()). // // We're reusing the same compiler for both evaluation paths here for brevity, // but it's likely a better fit to configure a separate compiler per use case. // === End Codelab === - return builder->Build(); + return std::move(builder)->Build(); } -// Parse a cel expression and evaluate it against the given activation and -// arena. -absl::StatusOr EvalCheckedExpr(const CheckedExpr& checked_expr, - const Activation& activation, - google::protobuf::Arena* arena) { - // Setup a default environment for building expressions. - InterpreterOptions options; - std::unique_ptr builder = CreateCelExpressionBuilder( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), options); - CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); - - // Note, the expression_plan below is reusable for different inputs, but we - // create one just in time for evaluation here. - CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&checked_expr)); - - CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, arena)); - - if (bool value; result.GetValue(&value)) { - return value; - } else if (const CelError * value; result.GetValue(&value)) { - return *value; - } else { - return absl::InvalidArgumentError(absl::StrCat( - "expected 'bool' result got '", result.DebugString(), "'")); +// Evaluate a runtime cel::Ast against the given activation and arena. +absl::StatusOr EvalAst(std::unique_ptr ast, + const cel::Activation& activation, + google::protobuf::Arena* arena) { + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN(cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + google::protobuf::DescriptorPool::generated_pool(), options)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + CEL_ASSIGN_OR_RETURN(cel::Value result, program->Evaluate(arena, activation)); + + if (result.IsBool()) { + return result.GetBool().NativeValue(); } + if (result.IsError()) { + return result.GetError().ToStatus(); + } + return absl::InvalidArgumentError( + absl::StrCat("expected 'bool' result got '", result.GetTypeName(), "'")); } + } // namespace absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, @@ -111,16 +102,21 @@ absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeCelCompiler()); - CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, - CompileToCheckedExpr(*compiler, cel_expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - Activation activation; + cel::Activation activation; google::protobuf::Arena arena; // === Start Codelab === - // Update the activation to bind the bool argument to 'bool_var' + // Update the activation to bind the bool argument to 'bool_var': + // activation.InsertOrAssignValue("bool_var", cel::BoolValue(bool_var)); // === End Codelab === - - return EvalCheckedExpr(checked_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } absl::StatusOr CompileAndEvaluateWithContext( @@ -128,16 +124,22 @@ absl::StatusOr CompileAndEvaluateWithContext( CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeCelCompiler()); - CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, - CompileToCheckedExpr(*compiler, cel_expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - Activation activation; + cel::Activation activation; google::protobuf::Arena arena; // === Start Codelab === - // Update the activation to bind the AttributeContext. + // Update the activation to bind the AttributeContext using + // cel::BindProtoToActivation (in + // "runtime/bind_proto_to_activation.h"). // === End Codelab === - - return EvalCheckedExpr(checked_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } } // namespace cel_codelab diff --git a/codelab/exercise2.h b/codelab/exercise2.h index d4836dc2b..7e0b3b8c7 100644 --- a/codelab/exercise2.h +++ b/codelab/exercise2.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE1_H_ -#define THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE1_H_ +#ifndef THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE2_H_ +#define THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE2_H_ #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/statusor.h" @@ -37,4 +37,4 @@ absl::StatusOr CompileAndEvaluateWithContext( } // namespace cel_codelab -#endif // THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE1_H_ +#endif // THIRD_PARTY_CEL_CPP_CODELAB_EXERCISE2_H_ diff --git a/codelab/exercise4.cc b/codelab/exercise4.cc index cf02a88bd..ff3e38ece 100644 --- a/codelab/exercise4.cc +++ b/codelab/exercise4.cc @@ -15,25 +15,31 @@ #include "codelab/exercise4.h" #include +#include -#include "cel/expr/checked.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "codelab/cel_compiler.h" +#include "absl/types/optional.h" // IWYU pragma: keep, needed for codelab solution +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/decl.h" // IWYU pragma: keep +#include "common/type.h" // IWYU pragma: keep +#include "common/value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/activation_bind_helper.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" +#include "runtime/function_adapter.h" // IWYU pragma: keep +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -41,21 +47,25 @@ namespace cel_codelab { namespace { -using ::cel::expr::CheckedExpr; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::BindProtoToActivation; -using ::google::api::expr::runtime::CelError; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::google::rpc::context::AttributeContext; +absl::StatusOr ContainsExtensionFunction( + const cel::MapValue& map, const cel::StringValue& key, + const cel::Value& value, const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena) { + (void)map; + (void)key; + (void)value; + (void)descriptor_pool; + (void)message_factory; + (void)arena; + // Implement the contains function: check if key is in map and if its value + // matches `value`. + return absl::UnimplementedError("Not yet implemented"); +} + absl::StatusOr> MakeConfiguredCompiler() { - // Setup for handling for protobuf types. - // Using the generated descriptor pool is simpler to configure, but often - // adds more types than necessary. + // Setup for handling protobuf types. google::protobuf::LinkMessageReflection(); CEL_ASSIGN_OR_RETURN( std::unique_ptr builder, @@ -67,48 +77,54 @@ absl::StatusOr> MakeConfiguredCompiler() { // Codelab part 1: // Add a declaration for the map.contains(string, V) function. - // Hint: use cel::MakeFunctionDecl and cel::TypeCheckerBuilder::MergeFunction. - return builder->Build(); + // Hint: use cel::MakeFunctionDecl and cel::MakeMemberOverloadDecl. + return std::move(builder)->Build(); } class Evaluator { public: - Evaluator() { - builder_ = CreateCelExpressionBuilder( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), options_); - } + Evaluator() = default; absl::Status SetupEvaluatorEnvironment() { - CEL_RETURN_IF_ERROR(RegisterBuiltinFunctions(builder_->GetRegistry())); + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + google::protobuf::DescriptorPool::generated_pool(), options)); // Codelab part 2: // Register the map.contains(string, value) function. - // Hint: use `CelFunctionAdapter::CreateAndRegister` to adapt from a free - // function ContainsExtensionFunction. + // Hint: use `TernaryFunctionAdapter::RegisterMemberOverload` to adapt + // from a free function ContainsExtensionFunction. + CEL_ASSIGN_OR_RETURN(runtime_, std::move(runtime_builder).Build()); return absl::OkStatus(); } - absl::StatusOr Evaluate(const CheckedExpr& expr, + absl::StatusOr Evaluate(std::unique_ptr ast, const AttributeContext& context) { - Activation activation; - CEL_RETURN_IF_ERROR(BindProtoToActivation(&context, &arena_, &activation)); - CEL_ASSIGN_OR_RETURN(auto plan, builder_->CreateExpression(&expr)); - CEL_ASSIGN_OR_RETURN(CelValue result, plan->Evaluate(activation, &arena_)); - - if (bool value; result.GetValue(&value)) { - return value; - } else if (const CelError * value; result.GetValue(&value)) { - return *value; - } else { - return absl::InvalidArgumentError( - absl::StrCat("unexpected return type: ", result.DebugString())); + cel::Activation activation; + CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), &arena_, &activation)); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime_->CreateProgram(std::move(ast))); + CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena_, activation)); + + if (result.IsBool()) { + return result.GetBool().NativeValue(); } + if (result.IsError()) { + return result.GetError().ToStatus(); + } + return absl::InvalidArgumentError( + absl::StrCat("unexpected return type: ", result.GetTypeName())); } private: google::protobuf::Arena arena_; - std::unique_ptr builder_; - InterpreterOptions options_; + std::unique_ptr runtime_; }; } // namespace @@ -118,15 +134,20 @@ absl::StatusOr EvaluateWithExtensionFunction( // Prepare a checked expression. CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeConfiguredCompiler()); - CEL_ASSIGN_OR_RETURN(auto checked_expr, - CompileToCheckedExpr(*compiler, expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } // Prepare an evaluation environment. Evaluator evaluator; CEL_RETURN_IF_ERROR(evaluator.SetupEvaluatorEnvironment()); - // Evaluate a checked expression against a particular activation - return evaluator.Evaluate(checked_expr, context); + // Evaluate the checked AST against a particular activation. + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return evaluator.Evaluate(std::move(ast), context); } } // namespace cel_codelab diff --git a/codelab/solutions/BUILD b/codelab/solutions/BUILD index a1597e182..d86ae8fe5 100644 --- a/codelab/solutions/BUILD +++ b/codelab/solutions/BUILD @@ -24,18 +24,22 @@ cc_library( srcs = ["exercise1.cc"], hdrs = ["//codelab:exercise1.h"], deps = [ - "//eval/public:activation", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", + "//checker:validation_result", + "//common:ast", + "//common:minimal_descriptor_pool", + "//common:value", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//internal:status_macros", - "//parser", + "//runtime", + "//runtime:activation", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -56,25 +60,24 @@ cc_library( hdrs = ["//codelab:exercise2.h"], deps = [ "//checker:type_checker_builder", - "//codelab:cel_compiler", + "//checker:validation_result", + "//common:ast", "//common:decl", "//common:type", + "//common:value", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:activation_bind_helper", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_options", - "//eval/public:cel_value", "//internal:status_macros", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", ], @@ -110,26 +113,27 @@ cc_library( srcs = ["exercise4.cc"], hdrs = ["//codelab:exercise4.h"], deps = [ - "//codelab:cel_compiler", + "//checker:type_checker_builder", + "//checker:validation_result", + "//common:ast", "//common:decl", "//common:type", + "//common:value", "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", - "//eval/public:activation", - "//eval/public:activation_bind_helper", - "//eval/public:builtin_func_registrar", - "//eval/public:cel_expr_builder_factory", - "//eval/public:cel_expression", - "//eval/public:cel_function_adapter", - "//eval/public:cel_options", - "//eval/public:cel_value", "//internal:status_macros", + "//runtime", + "//runtime:activation", + "//runtime:bind_proto_to_activation", + "//runtime:function_adapter", + "//runtime:runtime_builder", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", - "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", ], @@ -154,6 +158,7 @@ cc_library( deps = [ "//checker:validation_result", "//codelab:network_functions", + "//common:ast", "//common:decl", "//common:minimal_descriptor_pool", "//common:type", @@ -161,6 +166,7 @@ cc_library( "//compiler", "//compiler:compiler_factory", "//compiler:standard_library", + "//internal:status_macros", "//runtime", "//runtime:activation", "//runtime:runtime_builder", diff --git a/codelab/solutions/exercise1.cc b/codelab/solutions/exercise1.cc index aef6c0efe..e2fc4443a 100644 --- a/codelab/solutions/exercise1.cc +++ b/codelab/solutions/exercise1.cc @@ -16,90 +16,86 @@ #include #include +#include -#include "cel/expr/syntax.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "eval/public/activation.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" +#include "checker/validation_result.h" +#include "common/ast.h" +#include "common/minimal_descriptor_pool.h" +#include "common/value.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" #include "internal/status_macros.h" -#include "parser/parser.h" +#include "runtime/activation.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" namespace cel_codelab { namespace { -using ::cel::expr::ParsedExpr; -using ::google::api::expr::parser::Parse; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelExpression; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; - -// Convert the CelResult to a C++ string if it is string typed. Otherwise, -// return invalid argument error. This takes a copy to avoid lifecycle concerns -// (the evaluator may represent strings as stringviews backed by the input -// expression). -absl::StatusOr ConvertResult(const CelValue& value) { - if (CelValue::StringHolder inner_value; value.GetValue(&inner_value)) { - return std::string(inner_value.value()); - } else { - return absl::InvalidArgumentError(absl::StrCat( - "expected string result got '", CelValue::TypeName(value.type()), "'")); +// Convert the cel::Value to a C++ string if it is string typed. Otherwise, +// return invalid argument error. +absl::StatusOr ConvertResult(const cel::Value& value) { + if (value.IsString()) { + return value.GetString().ToString(); } + return absl::InvalidArgumentError( + absl::StrCat("expected string result got '", value.GetTypeName(), "'")); } + } // namespace absl::StatusOr ParseAndEvaluate(absl::string_view cel_expr) { // === Start Codelab === - // Setup a default environment for building expressions. - InterpreterOptions options; - std::unique_ptr builder = - CreateCelExpressionBuilder(options); - + // Setup a default compiler for compiling expressions. + CEL_ASSIGN_OR_RETURN( + std::unique_ptr compiler_builder, + cel::NewCompilerBuilder(cel::GetMinimalDescriptorPool())); CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); + compiler_builder->AddLibrary(cel::StandardCompilerLibrary())); + CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, + std::move(compiler_builder)->Build()); - // Parse the expression. This is fine for codelabs, but this skips the type - // checking phase. It won't check that functions and variables are available - // in the environment, and it won't handle certain ambiguous identifier - // expressions (e.g. container lookup vs namespaced name, packaged function - // vs. receiver call style function). - ParsedExpr parsed_expr; - CEL_ASSIGN_OR_RETURN(parsed_expr, Parse(cel_expr)); + // Compile the expression. + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - // The evaluator uses a proto Arena for incidental allocations during - // evaluation. + // Setup a standard runtime for evaluating expressions. + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN(cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + cel::GetMinimalDescriptorPool(), options)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + // Build the executable program from the compiled AST. + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + // The evaluator uses a proto Arena for allocations during evaluation. google::protobuf::Arena arena; - // The activation provides variables and functions that are bound into the + // The activation provides variables and functions bound into the // expression environment. In this example, there's no context expected, so - // we just provide an empty one to the evaluator. - Activation activation; - - // Build the expression plan. This assumes that the source expression AST and - // the expression builder outlive the CelExpression object. - CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); + // we provide an empty activation. + cel::Activation activation; - // Actually run the expression plan. We don't support any environment - // variables at the moment so just use an empty activation. - CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, &arena)); + // Run the program. + CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena, activation)); - // Convert the result to a c++ string. CelValues may reference instances from - // either the input expression, or objects allocated on the arena, so we need - // to pass ownership (in this case by copying to a new instance and returning - // that). + // Convert the result to a C++ string. return ConvertResult(result); // === End Codelab === } diff --git a/codelab/solutions/exercise10.cc b/codelab/solutions/exercise10.cc index 0d2c197d6..6d13652da 100644 --- a/codelab/solutions/exercise10.cc +++ b/codelab/solutions/exercise10.cc @@ -23,6 +23,7 @@ #include "absl/strings/string_view.h" #include "checker/validation_result.h" #include "codelab/network_functions.h" +#include "common/ast.h" #include "common/decl.h" #include "common/minimal_descriptor_pool.h" #include "common/type.h" @@ -30,6 +31,7 @@ #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" +#include "internal/status_macros.h" #include "runtime/activation.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" @@ -105,8 +107,9 @@ absl::StatusOr CompileAndEvaluateExercise10(absl::string_view expression, return absl::InvalidArgumentError(checked->FormatError()); } + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, checked->ReleaseAst()); absl::StatusOr> program = - (*runtime)->CreateProgram(checked->ReleaseAst().value()); + (*runtime)->CreateProgram(std::move(ast)); if (!program.ok()) { return std::move(program).status(); diff --git a/codelab/solutions/exercise2.cc b/codelab/solutions/exercise2.cc index d07645aed..9a2958e7d 100644 --- a/codelab/solutions/exercise2.cc +++ b/codelab/solutions/exercise2.cc @@ -15,29 +15,29 @@ #include "codelab/exercise2.h" #include +#include -#include "cel/expr/checked.pb.h" -#include "cel/expr/syntax.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "checker/type_checker_builder.h" -#include "codelab/cel_compiler.h" +#include "checker/validation_result.h" +#include "common/ast.h" #include "common/decl.h" #include "common/type.h" +#include "common/value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/activation_bind_helper.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -45,16 +45,6 @@ namespace cel_codelab { namespace { -using ::cel::expr::CheckedExpr; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::CelError; -using ::google::api::expr::runtime::CelExpression; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::ProtoUnsetFieldOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::google::rpc::context::AttributeContext; absl::StatusOr> MakeCelCompiler() { @@ -75,39 +65,38 @@ absl::StatusOr> MakeCelCompiler() { AttributeContext::descriptor()->full_name())); // === End Codelab === - return builder->Build(); + return std::move(builder)->Build(); } -// Parse a cel expression and evaluate it against the given activation and -// arena. -absl::StatusOr EvalCheckedExpr(const CheckedExpr& checked_expr, - const Activation& activation, - google::protobuf::Arena* arena) { - // Setup a default environment for building expressions. - InterpreterOptions options; - std::unique_ptr builder = CreateCelExpressionBuilder( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), options); - CEL_RETURN_IF_ERROR( - RegisterBuiltinFunctions(builder->GetRegistry(), options)); - - // Note, the expression_plan below is reusable for different inputs, but we +// Evaluate a runtime cel::Ast against the given activation and arena. +absl::StatusOr EvalAst(std::unique_ptr ast, + const cel::Activation& activation, + google::protobuf::Arena* arena) { + // Setup a default standard runtime. + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN(cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + google::protobuf::DescriptorPool::generated_pool(), options)); + CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + + // Note, the program plan below is reusable across different inputs, but we // create one just in time for evaluation here. - CEL_ASSIGN_OR_RETURN(std::unique_ptr expression_plan, - builder->CreateExpression(&checked_expr)); - - CEL_ASSIGN_OR_RETURN(CelValue result, - expression_plan->Evaluate(activation, arena)); - - if (bool value; result.GetValue(&value)) { - return value; - } else if (const CelError * value; result.GetValue(&value)) { - return *value; - } else { - return absl::InvalidArgumentError(absl::StrCat( - "expected 'bool' result got '", result.DebugString(), "'")); + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + CEL_ASSIGN_OR_RETURN(cel::Value result, program->Evaluate(arena, activation)); + + if (result.IsBool()) { + return result.GetBool().NativeValue(); + } + if (result.IsError()) { + return result.GetError().ToStatus(); } + return absl::InvalidArgumentError( + absl::StrCat("expected 'bool' result got '", result.GetTypeName(), "'")); } + } // namespace absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, @@ -115,16 +104,21 @@ absl::StatusOr CompileAndEvaluateWithBoolVar(absl::string_view cel_expr, CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeCelCompiler()); - CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, - CompileToCheckedExpr(*compiler, cel_expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - Activation activation; + cel::Activation activation; google::protobuf::Arena arena; // === Start Codelab === - activation.InsertValue("bool_var", CelValue::CreateBool(bool_var)); + activation.InsertOrAssignValue("bool_var", cel::BoolValue(bool_var)); // === End Codelab === - return EvalCheckedExpr(checked_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } absl::StatusOr CompileAndEvaluateWithContext( @@ -132,17 +126,24 @@ absl::StatusOr CompileAndEvaluateWithContext( CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeCelCompiler()); - CEL_ASSIGN_OR_RETURN(CheckedExpr checked_expr, - CompileToCheckedExpr(*compiler, cel_expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(cel_expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } - Activation activation; + cel::Activation activation; google::protobuf::Arena arena; // === Start Codelab === - CEL_RETURN_IF_ERROR(BindProtoToActivation( - &context, &arena, &activation, ProtoUnsetFieldOptions::kBindDefault)); + CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), &arena, &activation)); // === End Codelab === - return EvalCheckedExpr(checked_expr, activation, &arena); + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return EvalAst(std::move(ast), activation, &arena); } } // namespace cel_codelab diff --git a/codelab/solutions/exercise4.cc b/codelab/solutions/exercise4.cc index 244fdac05..bbeaff83b 100644 --- a/codelab/solutions/exercise4.cc +++ b/codelab/solutions/exercise4.cc @@ -15,29 +15,31 @@ #include "codelab/exercise4.h" #include +#include -#include "cel/expr/checked.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "codelab/cel_compiler.h" +#include "checker/type_checker_builder.h" +#include "checker/validation_result.h" +#include "common/ast.h" #include "common/decl.h" #include "common/type.h" +#include "common/value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/standard_library.h" -#include "eval/public/activation.h" -#include "eval/public/activation_bind_helper.h" -#include "eval/public/builtin_func_registrar.h" -#include "eval/public/cel_expr_builder_factory.h" -#include "eval/public/cel_expression.h" -#include "eval/public/cel_function_adapter.h" -#include "eval/public/cel_options.h" -#include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" +#include "runtime/function_adapter.h" +#include "runtime/runtime.h" +#include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -45,40 +47,24 @@ namespace cel_codelab { namespace { -using ::cel::expr::CheckedExpr; -using ::google::api::expr::runtime::Activation; -using ::google::api::expr::runtime::BindProtoToActivation; -using ::google::api::expr::runtime::CelError; -using ::google::api::expr::runtime::CelExpressionBuilder; -using ::google::api::expr::runtime::CelMap; -using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::CreateCelExpressionBuilder; -using ::google::api::expr::runtime::FunctionAdapter; -using ::google::api::expr::runtime::InterpreterOptions; -using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::google::rpc::context::AttributeContext; -// Handle the parametric type overload with a single generic CelValue overload. -absl::StatusOr ContainsExtensionFunction(google::protobuf::Arena* arena, - const CelMap* map, - CelValue::StringHolder key, - const CelValue& value) { - absl::optional entry = (*map)[CelValue::CreateString(key)]; +absl::StatusOr ContainsExtensionFunction( + const cel::MapValue& map, const cel::StringValue& key, + const cel::Value& value, const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN(absl::optional entry, + map.Find(key, descriptor_pool, message_factory, arena)); if (!entry.has_value()) { return false; } - if (value.IsInt64() && entry->IsInt64()) { - return value.Int64OrDie() == entry->Int64OrDie(); - } else if (value.IsString() && entry->IsString()) { - return value.StringOrDie().value() == entry->StringOrDie().value(); - } - return false; + CEL_ASSIGN_OR_RETURN(cel::Value equal, entry->Equal(value, descriptor_pool, + message_factory, arena)); + return equal.IsBool() && equal.GetBool().NativeValue(); } absl::StatusOr> MakeConfiguredCompiler() { - // Setup for handling for protobuf types. - // Using the generated descriptor pool is simpler to configure, but often - // adds more types than necessary. + // Setup for handling protobuf types. google::protobuf::LinkMessageReflection(); CEL_ASSIGN_OR_RETURN( std::unique_ptr builder, @@ -99,59 +85,66 @@ absl::StatusOr> MakeConfiguredCompiler() { cel::MakeFunctionDecl( "contains", cel::MakeMemberOverloadDecl( - "map_contains_string_string", cel::BoolType(), + "map_contains_string_value", cel::BoolType(), cel::MapType(checker_builder.arena(), cel::StringType(), cel::TypeParamType("V")), cel::StringType(), cel::TypeParamType("V")))); // Note: we use MergeFunction instead of AddFunction because we are adding // an overload to an already declared function with the same name. CEL_RETURN_IF_ERROR(checker_builder.MergeFunction(decl)); - return builder->Build(); + return std::move(builder)->Build(); } class Evaluator { public: - Evaluator() { - builder_ = CreateCelExpressionBuilder( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), options_); - } + Evaluator() = default; absl::Status SetupEvaluatorEnvironment() { - CEL_RETURN_IF_ERROR(RegisterBuiltinFunctions(builder_->GetRegistry())); + cel::RuntimeOptions options; + CEL_ASSIGN_OR_RETURN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder( + google::protobuf::DescriptorPool::generated_pool(), options)); // Codelab part 2: - // Register the map.contains(string, string) function. - // Hint: use `FunctionAdapter::CreateAndRegister` to adapt from a free - // function ContainsExtensionFunction. - using AdapterT = FunctionAdapter, const CelMap*, - CelValue::StringHolder, CelValue>; - CEL_RETURN_IF_ERROR(AdapterT::CreateAndRegister( - "contains", /*receiver_style=*/true, &ContainsExtensionFunction, - builder_->GetRegistry())); + // Register the map.contains(string, value) function. + // Hint: use `TernaryFunctionAdapter::RegisterMemberOverload` to adapt + // from a free function ContainsExtensionFunction. + using AdapterT = + cel::TernaryFunctionAdapter, const cel::MapValue&, + const cel::StringValue&, const cel::Value&>; + CEL_RETURN_IF_ERROR( + AdapterT::RegisterMemberOverload("contains", &ContainsExtensionFunction, + runtime_builder.function_registry())); + CEL_ASSIGN_OR_RETURN(runtime_, std::move(runtime_builder).Build()); return absl::OkStatus(); } - absl::StatusOr Evaluate(const CheckedExpr& expr, + absl::StatusOr Evaluate(std::unique_ptr ast, const AttributeContext& context) { - Activation activation; - CEL_RETURN_IF_ERROR(BindProtoToActivation(&context, &arena_, &activation)); - CEL_ASSIGN_OR_RETURN(auto plan, builder_->CreateExpression(&expr)); - CEL_ASSIGN_OR_RETURN(CelValue result, plan->Evaluate(activation, &arena_)); - - if (bool value; result.GetValue(&value)) { - return value; - } else if (const CelError* value; result.GetValue(&value)) { - return *value; - } else { - return absl::InvalidArgumentError( - absl::StrCat("unexpected return type: ", result.DebugString())); + cel::Activation activation; + CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), &arena_, &activation)); + + CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime_->CreateProgram(std::move(ast))); + CEL_ASSIGN_OR_RETURN(cel::Value result, + program->Evaluate(&arena_, activation)); + + if (result.IsBool()) { + return result.GetBool().NativeValue(); } + if (result.IsError()) { + return result.GetError().ToStatus(); + } + return absl::InvalidArgumentError( + absl::StrCat("unexpected return type: ", result.GetTypeName())); } private: google::protobuf::Arena arena_; - std::unique_ptr builder_; - InterpreterOptions options_; + std::unique_ptr runtime_; }; } // namespace @@ -161,15 +154,20 @@ absl::StatusOr EvaluateWithExtensionFunction( // Prepare a checked expression. CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler, MakeConfiguredCompiler()); - CEL_ASSIGN_OR_RETURN(auto checked_expr, - CompileToCheckedExpr(*compiler, expr)); + CEL_ASSIGN_OR_RETURN(cel::ValidationResult validation_result, + compiler->Compile(expr)); + if (!validation_result.IsValid()) { + return absl::InvalidArgumentError(validation_result.FormatError()); + } // Prepare an evaluation environment. Evaluator evaluator; CEL_RETURN_IF_ERROR(evaluator.SetupEvaluatorEnvironment()); - // Evaluate a checked expression against a particular activation - return evaluator.Evaluate(checked_expr, context); + // Evaluate the checked AST against a particular activation. + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + validation_result.ReleaseAst()); + return evaluator.Evaluate(std::move(ast), context); } } // namespace cel_codelab From 24e82b6de1837d99877fe502d7d7a2dfccc34d28 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 15 Jul 2026 11:39:32 -0700 Subject: [PATCH 121/350] Add brief intro on adding cel-cpp as a bazel dependency. Closes #2111 PiperOrigin-RevId: 948457033 --- codelab/README.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/codelab/README.md b/codelab/README.md index b20a5db31..c751d5678 100644 --- a/codelab/README.md +++ b/codelab/README.md @@ -31,6 +31,12 @@ Buffers as the input into CEL, they may be harder to understand. Consider working through one of these tutorials, first. See the devsite for [Protocol Buffers](https://protobuf.dev). +If you're not familiar with Bazel, Bazel is an open-source build and test tool +that supports multi-language projects and manages dependencies. This codelab and +`cel-cpp` are built using Bazel. To learn more about building C++ projects with +Bazel, see the official [Bazel C++ Tutorial](https://bazel.build/tutorials/cpp) +and the [Bazel Documentation](https://bazel.build/docs). + Notes on portability: Protocol Buffers are not required to use CEL generally, but the C++ implementation has a hard dependency on the library and some APIs reference protobuf types directly. Automated builds test against gcc10 and @@ -68,6 +74,62 @@ Make sure everything is working by building the codelab: bazel build //codelab:all ``` +## Setting up a Bazel Project with cel-cpp + +If you want to integrate `cel-cpp` into your own standalone C++ project using +[Bazel](https://bazel.build/docs), you can manage it as an external dependency +with [Bzlmod](https://bazel.build/external/overview#bzlmod). + +### MODULE.bazel + +In your project's root `MODULE.bazel` file, declare a dependency on `cel-cpp` +from the +[Bazel Central Registry (BCR)](https://registry.bazel.build/modules/cel-cpp): + +```python +bazel_dep(name = "cel-cpp", version = "") +``` + +Alternatively, if you want to depend directly on a specific commit from the +GitHub repository, you can use `git_override`: + +```python +bazel_dep(name = "cel-cpp", version = "0.0.0") +git_override( + module_name = "cel-cpp", + remote = "https://github.com/google/cel-cpp.git", + commit = "", +) +``` + +### BUILD + +In your project's `BUILD` (or `BUILD.bazel`) file, reference `cel-cpp` targets +using `@cel-cpp//...`. For example, to compile a C++ binary that compiles and +evaluates CEL expressions using the modern `Compiler` and `Runtime` APIs shown +in this codelab: + +```python +cc_binary( + name = "my_cel_app", + srcs = ["main.cc"], + deps = [ + "@cel-cpp//checker:validation_result", + "@cel-cpp//common:ast", + "@cel-cpp//common:minimal_descriptor_pool", + "@cel-cpp//common:value", + "@cel-cpp//compiler", + "@cel-cpp//compiler:compiler_factory", + "@cel-cpp//compiler:standard_library", + "@cel-cpp//runtime", + "@cel-cpp//runtime:activation", + "@cel-cpp//runtime:runtime_builder", + "@cel-cpp//runtime:runtime_options", + "@cel-cpp//runtime:standard_runtime_builder_factory", + ], +) +``` + ## Hello, World In the tried and true tradition of all programming languages, let's start with From 63cba4b26da8bd8ecb914a9c4fc573d353bbded1 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 15 Jul 2026 11:54:47 -0700 Subject: [PATCH 122/350] Add migration guide for CelValue -> cel::Value. Closes #1520 PiperOrigin-RevId: 948465671 --- eval/README.md | 241 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) diff --git a/eval/README.md b/eval/README.md index 32fa4bda4..bc805cb29 100644 --- a/eval/README.md +++ b/eval/README.md @@ -2,5 +2,246 @@ A C++ implementation of a [Common Expression Language][1] evaluator. +## Migrating to cel::Value APIs + +New users should prefer using the `cel::Value` APIs ("Modern"). The +`google::api::expr::runtime::CelValue` APIs ("Legacy") are not formally +deprecated at this time, but they now incur some overhead and will not be +updated to support all new features. Internally, both flavors use the same +underlying implementation based on the cel::Value representation. + +### When to Migrate + +If your integration is stable and you don't need access to newer features there +is no need to migrate at this time. + +If you need to add support for optionals or other custom opaque or struct types +you must migrate. + +If you need to interactively evaluate from source, you should migrate to avoid +extra proto serialization costs. + +### How to Migrate + +#### 1. Header, Namespace, and Build Target Mapping + +Concept | Legacy API (`google::api::expr::runtime`) | Modern API (`cel`) +:----------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------- +**Namespace** | `google::api::expr::runtime` | `cel` +**Runtime & Program Creation** | `#include "eval/public/cel_expression.h"`
`#include "eval/public/cel_expr_builder_factory.h"`
Target: `//eval/public:cel_expression` | `#include "runtime/runtime.h"`
`#include "runtime/standard_runtime_builder_factory.h"`
Targets: `//runtime`, `//runtime:standard_runtime_builder_factory` +**Value Representation** | `#include "eval/public/cel_value.h"`
Target: `//eval/public:cel_value` | `#include "common/value.h"`
Target: `//common:value` +**Activation & Variables** | `#include "eval/public/activation.h"`
Target: `//eval/public:activation` | `#include "runtime/activation.h"`
Target: `//runtime:activation` +**Proto Activation Binding** | `#include "eval/public/activation_bind_helper.h"`
Target: `//eval/public:activation_bind_helper` | `#include "runtime/bind_proto_to_activation.h"`
Target: `//runtime:bind_proto_to_activation` +**Function Adapters** | `#include "eval/public/cel_function_adapter.h"` | `#include "runtime/function_adapter.h"` +**Interoperability Adapters** | N/A | `#include "common/legacy_value.h"`
Target: `//common:legacy_value` + +#### 2. Runtime Creation and AST Planning (`CelExpressionBuilder` vs `cel::Runtime`) + +In the legacy API, `CelExpressionBuilder` created expression plans from +`google::api::expr::CheckedExpr` protobufs. In the modern API, `cel::Runtime` +creates reusable `cel::Program` instances directly from `cel::Ast` objects +without requiring protobuf serialization round-trips. + +**Legacy (`google::api::expr::runtime::CelExpressionBuilder`)**: + +```cpp +using ::google::api::expr::runtime::CelExpressionBuilder; +using ::google::api::expr::runtime::CreateCelExpressionBuilder; +using ::google::api::expr::runtime::InterpreterOptions; +using ::google::api::expr::runtime::RegisterBuiltinFunctions; + +InterpreterOptions options; +std::unique_ptr builder = + CreateCelExpressionBuilder(descriptor_pool, message_factory, options); +CEL_RETURN_IF_ERROR(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + +// Requires serializing or passing a google::api::expr::CheckedExpr proto +CEL_ASSIGN_OR_RETURN(std::unique_ptr plan, + builder->CreateExpression(&checked_expr_proto)); +``` + +**Modern (`cel::Runtime`)**: + +```cpp +#include "runtime/runtime.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" + +cel::RuntimeOptions options; +CEL_ASSIGN_OR_RETURN( + cel::RuntimeBuilder runtime_builder, + cel::CreateStandardRuntimeBuilder(descriptor_pool, options)); +CEL_ASSIGN_OR_RETURN(std::unique_ptr runtime, + std::move(runtime_builder).Build()); + +// Create program directly from a cel::Ast (e.g., from compiler->Compile(expr)) +CEL_ASSIGN_OR_RETURN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); +``` + +*Note on AST Conversion*: If your architecture persists or receives serialized +`google::api::expr::CheckedExpr` protobufs, convert them to `cel::Ast` using +`cel::CreateAstFromCheckedExpr(checked_expr_proto)` (`#include +"common/ast_proto.h"`, target +`//common:ast_proto`). + +#### 3. Creating, Inspecting, and Unpacking Values (`CelValue` vs `cel::Value`) + +**Legacy (`google::api::expr::runtime::CelValue`)**: + +```cpp +#include "eval/public/cel_value.h" + +using ::google::api::expr::runtime::CelValue; + +// Creation +CelValue bool_val = CelValue::CreateBool(true); +CelValue int_val = CelValue::CreateInt64(42); +CelValue str_val = CelValue::CreateStringView("hello"); + +// Inspection & Unpacking +if (bool_val.IsBool()) { + bool b; + bool_val.GetValue(&b); +} +``` + +**Modern (`cel::Value`)**: + +```cpp +#include "common/value.h" + +// Creation +cel::Value bool_val = cel::BoolValue(true); +cel::Value int_val = cel::IntValue(42); +cel::Value str_val = cel::StringValue("hello"); + +// Inspection & Unpacking +if (bool_val.IsBool()) { + bool b = bool_val.GetBool().NativeValue(); +} else if (str_val.IsString()) { + std::string s = str_val.GetString().ToString(); +} else if (str_val.IsError()) { + absl::Status status = str_val.GetError().ToStatus(); +} +``` + +#### 4. Activation and Protobuf Binding (`BindProtoToActivation`) + +The modern equivalent of `activation_bind_helper.h` is +`runtime/bind_proto_to_activation.h`. + +**Legacy**: + +```cpp +#include "eval/public/activation.h" +#include "eval/public/activation_bind_helper.h" + +using ::google::api::expr::runtime::Activation; +using ::google::api::expr::runtime::BindProtoToActivation; +using ::google::api::expr::runtime::ProtoUnsetFieldOptions; + +Activation activation; +activation.InsertValue("bool_var", CelValue::CreateBool(true)); +CEL_RETURN_IF_ERROR(BindProtoToActivation( + &context_message, &arena, &activation, ProtoUnsetFieldOptions::kBindDefault)); +``` + +**Modern**: + +```cpp +#include "runtime/activation.h" +#include "runtime/bind_proto_to_activation.h" + +cel::Activation activation; +activation.InsertOrAssignValue("bool_var", cel::BoolValue(true)); +CEL_RETURN_IF_ERROR(cel::BindProtoToActivation( + context_message, cel::BindProtoUnsetFieldBehavior::kBindDefaultValue, + descriptor_pool, message_factory, &arena, &activation)); +``` + +#### 5. Program Evaluation + +Notice the parameter order change when calling `Evaluate`: `(&arena, +activation)` instead of `(activation, &arena)`. + +**Legacy**: + +```cpp +CEL_ASSIGN_OR_RETURN(CelValue result, plan->Evaluate(activation, &arena)); +``` + +**Modern**: + +```cpp +CEL_ASSIGN_OR_RETURN(cel::Value result, program->Evaluate(&arena, activation)); +``` + +#### 6. Custom Extension Functions (`FunctionAdapter`) + +**Legacy (`google::api::expr::runtime::FunctionAdapter`)**: + +```cpp +#include "eval/public/cel_function_adapter.h" + +using AdapterT = google::api::expr::runtime::FunctionAdapter< + absl::StatusOr, const CelMap*, CelValue::StringHolder, CelValue>; +CEL_RETURN_IF_ERROR(AdapterT::CreateAndRegister( + "contains", /*receiver_style=*/true, &ContainsFunc, builder->GetRegistry())); +``` + +**Modern (`cel::FunctionAdapter` / `cel::{arity}FunctionAdapter`)**: + +```cpp +#include "runtime/function_adapter.h" + +using AdapterT = cel::TernaryFunctionAdapter< + absl::StatusOr, const cel::MapValue&, const cel::StringValue&, const cel::Value&>; +CEL_RETURN_IF_ERROR(AdapterT::RegisterMemberOverload( + "contains", &ContainsFunc, runtime_builder.function_registry())); +``` + +### Other behavior notes + +There is limited support for interoperability when migrating incrementally. You +can use `#include "common/legacy_value.h"` (build target +`//common:legacy_value`) to adapt compatible value types +across legacy and modern boundaries: + +* **Adapting Legacy `CelValue` to Modern `cel::Value`**: + + ```cpp + #include "common/legacy_value.h" + + CEL_ASSIGN_OR_RETURN(cel::Value modern_val, + cel::ModernValue(&arena, legacy_cel_val)); + ``` + +* **Adapting Modern `cel::Value` to Legacy `CelValue`**: + + ```cpp + #include "common/legacy_value.h" + + CEL_ASSIGN_OR_RETURN(google::api::expr::runtime::CelValue legacy_val, + cel::LegacyValue(&arena, modern_val)); + ``` + +Legacy style extension functions use the interop helpers implicitly and can be +used with modern APIs. `google::api::expr::runtime::CelFunction` is also a +`cel::Function`. + +```cpp +std::unique_ptr ext_func = ...; + +cel::FunctionDescriptor descriptor = ext_func->descriptor(); +CEL_RETURN_IF_ERROR( + runtime_builder.function_registry().Register( + descriptor, std::move(ext_func))); +``` + +Important: The interop code does not support adapting new types introduced with +`cel::Value` to `google::api::expr::runtime::CelValue`. This means you should +not use any of the legacy APIs if your expressions refer to values with an +optional, custom (non-protobuf) struct, or custom opaque type. [1]: https://github.com/cel-expr/cel-spec From 070dfcb3451cac1acfe1d7f3d8d0ff62f5507e3b Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 17 Jul 2026 10:39:22 -0700 Subject: [PATCH 123/350] Refactor select step impl to behave consistently between iterative and recursive programs. PiperOrigin-RevId: 949647076 --- eval/compiler/flat_expr_builder.cc | 8 +- eval/eval/select_step.cc | 364 ++++++++++------------------- eval/eval/select_step.h | 3 +- eval/eval/select_step_test.cc | 25 +- 4 files changed, 149 insertions(+), 251 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index fc7289bd2..d1435b4a3 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -976,6 +976,7 @@ class FlatExprVisitor : public cel::AstVisitor { return; } + StringValue field = cel::StringValue(select_expr.field()); if (auto depth = RecursionEligible(); depth.has_value()) { auto deps = ExtractRecursiveDependencies(); if (deps.size() != 1) { @@ -983,7 +984,6 @@ class FlatExprVisitor : public cel::AstVisitor { "unexpected number of dependencies for select operation.")); return; } - StringValue field = cel::StringValue(select_expr.field()); SetRecursiveStep( CreateDirectSelectStep(std::move(deps[0]), std::move(field), @@ -994,9 +994,9 @@ class FlatExprVisitor : public cel::AstVisitor { return; } - AddStep(CreateSelectStep(select_expr, expr.id(), - options_.enable_empty_wrapper_null_unboxing, - enable_optional_types_)); + AddStep(CreateSelectStep( + std::move(field), select_expr.test_only(), expr.id(), + options_.enable_empty_wrapper_null_unboxing, enable_optional_types_)); } // Call node handler group. diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index b815f5d87..806e20b31 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -5,14 +5,12 @@ #include #include -#include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/expr.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" @@ -32,7 +30,6 @@ namespace { using ::cel::BoolValue; using ::cel::ErrorValue; using ::cel::MapValue; -using ::cel::NullValue; using ::cel::OptionalValue; using ::cel::ProtoWrapperTypeOptions; using ::cel::StringValue; @@ -75,35 +72,97 @@ absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, return std::nullopt; } -void TestOnlySelect(const StructValue& msg, const std::string& field, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, - Value* absl_nonnull result) { - absl::StatusOr has_field = msg.HasFieldByName(field); +absl::Status PerformHas(const Value& target, absl::string_view field, + const StringValue& field_value, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, Value& result) { + switch (target.kind()) { + case ValueKind::kMap: { + CEL_RETURN_IF_ERROR(target.GetMap().Has(field_value, descriptor_pool, + message_factory, arena, &result)); + return absl::OkStatus(); + } + case ValueKind::kStruct: { + auto has_field = target.GetStruct().HasFieldByName(field); + if (!has_field.ok()) { + result = ErrorValue(std::move(has_field).status()); + } else { + result = BoolValue{*has_field}; + } + return absl::OkStatus(); + } + default: + return InvalidSelectTargetError(); + } +} - if (!has_field.ok()) { - *result = ErrorValue(std::move(has_field).status()); - return; +absl::Status PerformGet(const Value& target, absl::string_view field, + const StringValue& field_value, + ProtoWrapperTypeOptions unboxing_option, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, Value& result) { + switch (target.kind()) { + case ValueKind::kMap: { + auto status = target.GetMap().Get(field_value, descriptor_pool, + message_factory, arena, &result); + if (!status.ok()) { + result = ErrorValue(std::move(status)); + } + return absl::OkStatus(); + } + case ValueKind::kStruct: { + auto status = target.GetStruct().GetFieldByName( + field, unboxing_option, descriptor_pool, message_factory, arena, + &result); + if (!status.ok()) { + result = ErrorValue(std::move(status)); + } + return absl::OkStatus(); + } + default: + return InvalidSelectTargetError(); } - *result = BoolValue{*has_field}; } -void TestOnlySelect(const MapValue& map, const StringValue& field_name, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, - Value* absl_nonnull result) { - // Field presence only supports string keys containing valid identifier - // characters. - absl::Status presence = - map.Has(field_name, descriptor_pool, message_factory, arena, result); - - if (!presence.ok()) { - *result = ErrorValue(std::move(presence)); - return; +absl::Status PerformOptionalGet(const Value& target, absl::string_view field, + const StringValue& field_value, + ProtoWrapperTypeOptions unboxing_option, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, Value& result) { + switch (target.kind()) { + case ValueKind::kMap: { + CEL_ASSIGN_OR_RETURN( + bool found, target.GetMap().Find(field_value, descriptor_pool, + message_factory, arena, &result)); + if (!found) { + result = OptionalValue::None(); + return absl::OkStatus(); + } + ABSL_DCHECK(!result.IsUnknown()); + result = OptionalValue::Of(std::move(result), arena); + return absl::OkStatus(); + } + case ValueKind::kStruct: { + CEL_ASSIGN_OR_RETURN(bool found, + target.GetStruct().HasFieldByName(field)); + if (!found) { + result = OptionalValue::None(); + return absl::OkStatus(); + } + CEL_RETURN_IF_ERROR(target.GetStruct().GetFieldByName( + field, unboxing_option, descriptor_pool, message_factory, arena, + &result)); + + ABSL_DCHECK(!result.IsUnknown()); + result = OptionalValue::Of(std::move(result), arena); + return absl::OkStatus(); + } + default: + return InvalidSelectTargetError(); } - ABSL_DCHECK(!result->IsUnknown()); } // SelectStep performs message field access specified by Expr::Select @@ -124,11 +183,6 @@ class SelectStep : public ExpressionStepBase { absl::Status Evaluate(ExecutionFrame* frame) const override; private: - absl::Status PerformTestOnlySelect(ExecutionFrame* frame, - const Value& arg) const; - absl::StatusOr PerformSelect(ExecutionFrame* frame, const Value& arg, - Value& result) const; - cel::StringValue field_value_; std::string field_; bool test_field_presence_; @@ -177,128 +231,48 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { return absl::OkStatus(); } - // Handle test only Select. + Value result; if (test_field_presence_) { + const Value* target = &arg; if (optional_arg) { if (!optional_arg->HasValue()) { - frame->value_stack().PopAndPush(cel::BoolValue{false}); + frame->value_stack().PopAndPush(cel::BoolValue{false}, + std::move(result_trail)); return absl::OkStatus(); } - Value value; - optional_arg->Value(&value); - return PerformTestOnlySelect(frame, value); + optional_arg->Value(&result); + target = &result; } - return PerformTestOnlySelect(frame, arg); + CEL_RETURN_IF_ERROR( + PerformHas(*target, field_, field_value_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), result)); + frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); + return absl::OkStatus(); } - // Normal select path. - // Select steps can be applied to either maps or messages if (optional_arg) { if (!optional_arg->HasValue()) { - // Leave optional_arg at the top of the stack. Its empty. + frame->value_stack().PopAndPush(OptionalValue::None(), + std::move(result_trail)); return absl::OkStatus(); } Value value; - Value result; - bool ok; optional_arg->Value(&value); - CEL_ASSIGN_OR_RETURN(ok, PerformSelect(frame, value, result)); - if (!ok) { - frame->value_stack().PopAndPush(cel::OptionalValue::None(), - std::move(result_trail)); - return absl::OkStatus(); + auto status = PerformOptionalGet( + value, field_, field_value_, unboxing_option_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), result); + if (!status.ok()) { + result = ErrorValue(std::move(status)); } - frame->value_stack().PopAndPush( - cel::OptionalValue::Of(std::move(result), frame->arena()), - std::move(result_trail)); + frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); return absl::OkStatus(); } - // Normal select path. - // Select steps can be applied to either maps or messages - switch (arg.kind()) { - case ValueKind::kStruct: { - Value result; - auto status = arg.GetStruct().GetFieldByName( - field_, unboxing_option_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result); - if (!status.ok()) { - result = ErrorValue(std::move(status)); - } - frame->value_stack().PopAndPush(std::move(result), - std::move(result_trail)); - return absl::OkStatus(); - } - case ValueKind::kMap: { - Value result; - auto status = - arg.GetMap().Get(field_value_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result); - if (!status.ok()) { - result = ErrorValue(std::move(status)); - } - frame->value_stack().PopAndPush(std::move(result), - std::move(result_trail)); - return absl::OkStatus(); - } - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); - } -} - -absl::Status SelectStep::PerformTestOnlySelect(ExecutionFrame* frame, - const Value& arg) const { - switch (arg.kind()) { - case ValueKind::kMap: { - Value result; - TestOnlySelect(arg.GetMap(), field_value_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result); - frame->value_stack().PopAndPush(std::move(result)); - return absl::OkStatus(); - } - case ValueKind::kMessage: { - Value result; - TestOnlySelect(arg.GetStruct(), field_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result); - frame->value_stack().PopAndPush(std::move(result)); - return absl::OkStatus(); - } - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); - } -} - -absl::StatusOr SelectStep::PerformSelect(ExecutionFrame* frame, - const Value& arg, - Value& result) const { - switch (arg->kind()) { - case ValueKind::kStruct: { - const auto& struct_value = arg.GetStruct(); - CEL_ASSIGN_OR_RETURN(auto ok, struct_value.HasFieldByName(field_)); - if (!ok) { - result = NullValue{}; - return false; - } - CEL_RETURN_IF_ERROR(struct_value.GetFieldByName( - field_, unboxing_option_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result)); - ABSL_DCHECK(!result.IsUnknown()); - return true; - } - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN( - auto found, - arg.GetMap().Find(field_value_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result)); - ABSL_DCHECK(!found || !result.IsUnknown()); - return found; - } - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); - } + CEL_RETURN_IF_ERROR(PerformGet( + arg, field_, field_value_, unboxing_option_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), result)); + frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); + return absl::OkStatus(); } class DirectSelectStep : public DirectExpressionStep { @@ -362,11 +336,11 @@ class DirectSelectStep : public DirectExpressionStep { } Value value; optional_arg->Value(&value); - PerformTestOnlySelect(frame, value, result); - return absl::OkStatus(); + return PerformHas(value, field_, field_value_, frame.descriptor_pool(), + frame.message_factory(), frame.arena(), result); } - PerformTestOnlySelect(frame, result, result); - return absl::OkStatus(); + return PerformHas(result, field_, field_value_, frame.descriptor_pool(), + frame.message_factory(), frame.arena(), result); } if (optional_arg) { @@ -376,26 +350,24 @@ class DirectSelectStep : public DirectExpressionStep { } Value value; optional_arg->Value(&value); - return PerformOptionalSelect(frame, value, result); + auto status = + PerformOptionalGet(value, field_, field_value_, unboxing_option_, + frame.descriptor_pool(), frame.message_factory(), + frame.arena(), result); + if (!status.ok()) { + result = ErrorValue(std::move(status)); + } + return absl::OkStatus(); } - auto status = PerformSelect(frame, result, result); - if (!status.ok()) { - result = ErrorValue(std::move(status)); - } - return absl::OkStatus(); + return PerformGet(result, field_, field_value_, unboxing_option_, + frame.descriptor_pool(), frame.message_factory(), + frame.arena(), result); } private: std::unique_ptr operand_; - void PerformTestOnlySelect(ExecutionFrameBase& frame, const Value& value, - Value& result) const; - absl::Status PerformOptionalSelect(ExecutionFrameBase& frame, - const Value& value, Value& result) const; - absl::Status PerformSelect(ExecutionFrameBase& frame, const Value& value, - Value& result) const; - // Field name in formats supported by each of the map and struct field access // APIs. // @@ -410,84 +382,6 @@ class DirectSelectStep : public DirectExpressionStep { bool enable_optional_types_; }; -void DirectSelectStep::PerformTestOnlySelect(ExecutionFrameBase& frame, - const cel::Value& value, - Value& result) const { - switch (value.kind()) { - case ValueKind::kMap: - TestOnlySelect(value.GetMap(), field_value_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result); - return; - case ValueKind::kMessage: - TestOnlySelect(value.GetStruct(), field_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result); - return; - default: - // Control flow should have returned earlier. - result = cel::ErrorValue(InvalidSelectTargetError()); - return; - } -} - -absl::Status DirectSelectStep::PerformOptionalSelect(ExecutionFrameBase& frame, - const Value& value, - Value& result) const { - switch (value.kind()) { - case ValueKind::kStruct: { - auto struct_value = value.GetStruct(); - CEL_ASSIGN_OR_RETURN(auto ok, struct_value.HasFieldByName(field_)); - if (!ok) { - result = OptionalValue::None(); - return absl::OkStatus(); - } - CEL_RETURN_IF_ERROR(struct_value.GetFieldByName( - field_, unboxing_option_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - ABSL_DCHECK(!result.IsUnknown()); - result = OptionalValue::Of(std::move(result), frame.arena()); - return absl::OkStatus(); - } - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN( - auto found, - value.GetMap().Find(field_value_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - if (!found) { - result = OptionalValue::None(); - return absl::OkStatus(); - } - ABSL_DCHECK(!result.IsUnknown()); - result = OptionalValue::Of(std::move(result), frame.arena()); - return absl::OkStatus(); - } - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); - } -} - -absl::Status DirectSelectStep::PerformSelect(ExecutionFrameBase& frame, - const cel::Value& value, - Value& result) const { - switch (value.kind()) { - case ValueKind::kStruct: - CEL_RETURN_IF_ERROR(value.GetStruct().GetFieldByName( - field_, unboxing_option_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - ABSL_DCHECK(!result.IsUnknown()); - return absl::OkStatus(); - case ValueKind::kMap: - CEL_RETURN_IF_ERROR( - value.GetMap().Get(field_value_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - ABSL_DCHECK(!result.IsUnknown()); - return absl::OkStatus(); - default: - // Control flow should have returned earlier. - return InvalidSelectTargetError(); - } -} - } // namespace std::unique_ptr CreateDirectSelectStep( @@ -501,11 +395,11 @@ std::unique_ptr CreateDirectSelectStep( // Factory method for Select - based Execution step absl::StatusOr> CreateSelectStep( - const cel::SelectExpr& select_expr, int64_t expr_id, + cel::StringValue field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { - return std::make_unique( - cel::StringValue(select_expr.field()), select_expr.test_only(), expr_id, - enable_wrapper_type_null_unboxing, enable_optional_types); + return std::make_unique(std::move(field), test_only, expr_id, + enable_wrapper_type_null_unboxing, + enable_optional_types); } } // namespace google::api::expr::runtime diff --git a/eval/eval/select_step.h b/eval/eval/select_step.h index 6eaaf9487..2fe35ecb0 100644 --- a/eval/eval/select_step.h +++ b/eval/eval/select_step.h @@ -5,7 +5,6 @@ #include #include "absl/status/statusor.h" -#include "common/expr.h" #include "common/value.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" @@ -20,7 +19,7 @@ std::unique_ptr CreateDirectSelectStep( // Factory method for Select - based Execution step absl::StatusOr> CreateSelectStep( - const cel::SelectExpr& select_expr, int64_t expr_id, + cel::StringValue field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types = false); } // namespace google::api::expr::runtime diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index ce532eabd..d5ca2aecd 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -133,8 +133,8 @@ class SelectStepTest : public testing::Test { CEL_ASSIGN_OR_RETURN(auto step0, CreateIdentStep(ident.name(), expr0.id())); CEL_ASSIGN_OR_RETURN( auto step1, - CreateSelectStep(select, expr.id(), - options.enable_wrapper_type_null_unboxing)); + CreateSelectStep(cel::StringValue(select.field()), select.test_only(), + expr.id(), options.enable_wrapper_type_null_unboxing)); path.push_back(std::move(step0)); path.push_back(std::move(step1)); @@ -330,11 +330,13 @@ TEST_F(SelectStepTest, MapPresenseIsErrorTest) { ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select_map, expr1.id(), + CreateSelectStep(cel::StringValue(select_map.field()), + select_map.test_only(), expr1.id(), /*enable_wrapper_type_null_unboxing=*/false)); ASSERT_OK_AND_ASSIGN( auto step2, - CreateSelectStep(select, select_expr.id(), + CreateSelectStep(cel::StringValue(select.field()), select.test_only(), + select_expr.id(), /*enable_wrapper_type_null_unboxing=*/false)); ExecutionPath path; @@ -836,7 +838,8 @@ TEST_P(SelectStepConformanceTest, CelErrorAsArgument) { ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select, dummy_expr.id(), + CreateSelectStep(cel::StringValue(select.field()), select.test_only(), + dummy_expr.id(), /*enable_wrapper_type_null_unboxing=*/false)); path.push_back(std::move(step0)); @@ -877,7 +880,8 @@ TEST_F(SelectStepTest, DisableMissingAttributeOK) { ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select, dummy_expr.id(), + CreateSelectStep(cel::StringValue(select.field()), select.test_only(), + dummy_expr.id(), /*enable_wrapper_type_null_unboxing=*/false)); path.push_back(std::move(step0)); @@ -919,7 +923,8 @@ TEST_F(SelectStepTest, UnrecoverableUnknownValueProducesError) { ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select, dummy_expr.id(), + CreateSelectStep(cel::StringValue(select.field()), select.test_only(), + dummy_expr.id(), /*enable_wrapper_type_null_unboxing=*/false)); path.push_back(std::move(step0)); @@ -965,9 +970,9 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); auto step0_status = CreateIdentStep(ident.name(), expr0.id()); - auto step1_status = - CreateSelectStep(select, dummy_expr.id(), - /*enable_wrapper_type_null_unboxing=*/false); + auto step1_status = CreateSelectStep( + cel::StringValue(select.field()), select.test_only(), dummy_expr.id(), + /*enable_wrapper_type_null_unboxing=*/false); ASSERT_THAT(step0_status, IsOk()); ASSERT_THAT(step1_status, IsOk()); From ff55aa8deaf6e3464281dbfe7430f1c056cc2a39 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Fri, 17 Jul 2026 19:27:35 -0700 Subject: [PATCH 124/350] No public description PiperOrigin-RevId: 949876625 --- eval/eval/shadowable_value_step.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/eval/shadowable_value_step.cc b/eval/eval/shadowable_value_step.cc index 1ebab2f1e..0839d28cc 100644 --- a/eval/eval/shadowable_value_step.cc +++ b/eval/eval/shadowable_value_step.cc @@ -85,8 +85,8 @@ absl::Status DirectShadowableValueStep::Evaluate( absl::StatusOr> CreateShadowableValueStep( absl::string_view name, cel::Value value, int64_t expr_id) { - return absl::make_unique(std::string(name), - std::move(value), expr_id); + return std::make_unique(std::string(name), + std::move(value), expr_id); } std::unique_ptr CreateDirectShadowableValueStep( From e9e97ee3fe09e65b43f54c1e1abf0cdcd0bd0718 Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Mon, 20 Jul 2026 10:24:17 -0700 Subject: [PATCH 125/350] Add validation for out-of-bounds durations and timestamps in ConvertToJson. PiperOrigin-RevId: 950916413 --- common/values/duration_value.cc | 2 ++ common/values/duration_value_test.cc | 13 +++++++++++++ common/values/timestamp_value.cc | 2 ++ common/values/timestamp_value_test.cc | 13 +++++++++++++ 4 files changed, 30 insertions(+) diff --git a/common/values/duration_value.cc b/common/values/duration_value.cc index a3b41e8ea..1f3d5b723 100644 --- a/common/values/duration_value.cc +++ b/common/values/duration_value.cc @@ -75,6 +75,8 @@ absl::Status DurationValue::ConvertToJson( ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); + CEL_RETURN_IF_ERROR(cel::internal::ValidateDuration(NativeValue())); + ValueReflection value_reflection; CEL_RETURN_IF_ERROR(value_reflection.Initialize(json->GetDescriptor())); value_reflection.SetStringValueFromDuration(json, NativeValue()); diff --git a/common/values/duration_value_test.cc b/common/values/duration_value_test.cc index 29d9b0f9e..d525f2497 100644 --- a/common/values/duration_value_test.cc +++ b/common/values/duration_value_test.cc @@ -12,9 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "common/values/duration_value.h" + #include #include +#include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/time/time.h" #include "common/native_type.h" @@ -66,6 +69,16 @@ TEST_F(DurationValueTest, ConvertToJson) { EXPECT_THAT(*message, EqualsValueTextProto(R"pb(string_value: "0s")pb")); } +TEST_F(DurationValueTest, ConvertToJsonOutOfBounds) { + auto* message = NewArenaValueMessage(); + EXPECT_THAT(UnsafeDurationValue(absl::InfiniteDuration()) + .ConvertToJson(descriptor_pool(), message_factory(), message), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(UnsafeDurationValue(-absl::InfiniteDuration()) + .ConvertToJson(descriptor_pool(), message_factory(), message), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument)); +} + TEST_F(DurationValueTest, NativeTypeId) { EXPECT_EQ(NativeTypeId::Of(DurationValue(absl::Seconds(1))), NativeTypeId::For()); diff --git a/common/values/timestamp_value.cc b/common/values/timestamp_value.cc index 7d3a347e8..0aff5f3d7 100644 --- a/common/values/timestamp_value.cc +++ b/common/values/timestamp_value.cc @@ -75,6 +75,8 @@ absl::Status TimestampValue::ConvertToJson( ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); + CEL_RETURN_IF_ERROR(cel::internal::ValidateTimestamp(NativeValue())); + ValueReflection value_reflection; CEL_RETURN_IF_ERROR(value_reflection.Initialize(json->GetDescriptor())); value_reflection.SetStringValueFromTimestamp(json, NativeValue()); diff --git a/common/values/timestamp_value_test.cc b/common/values/timestamp_value_test.cc index 142e6511d..ff34c8000 100644 --- a/common/values/timestamp_value_test.cc +++ b/common/values/timestamp_value_test.cc @@ -12,8 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "common/values/timestamp_value.h" + #include +#include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/time/time.h" #include "common/native_type.h" @@ -56,6 +59,16 @@ TEST_F(TimestampValueTest, ConvertToJson) { R"pb(string_value: "1970-01-01T00:00:00Z")pb")); } +TEST_F(TimestampValueTest, ConvertToJsonOutOfBounds) { + auto* message = NewArenaValueMessage(); + EXPECT_THAT(UnsafeTimestampValue(absl::InfiniteFuture()) + .ConvertToJson(descriptor_pool(), message_factory(), message), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(UnsafeTimestampValue(absl::InfinitePast()) + .ConvertToJson(descriptor_pool(), message_factory(), message), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument)); +} + TEST_F(TimestampValueTest, NativeTypeId) { EXPECT_EQ( NativeTypeId::Of(TimestampValue(absl::UnixEpoch() + absl::Seconds(1))), From 8d2bf66b16fa53233c8c4b177de45d659abac4a4 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 20 Jul 2026 11:14:56 -0700 Subject: [PATCH 126/350] Refactor: parameter order for ConvertTypeSpecToType For internal consistency (output parameter last). PiperOrigin-RevId: 950948076 --- common/signature.cc | 2 +- common/signature_test.cc | 9 +++---- common/type_spec_resolver.cc | 18 +++++++------- common/type_spec_resolver.h | 4 ++-- common/type_spec_resolver_test.cc | 39 ++++++++++++++++--------------- 5 files changed, 37 insertions(+), 35 deletions(-) diff --git a/common/signature.cc b/common/signature.cc index 54d312777..3f4ea8d29 100644 --- a/common/signature.cc +++ b/common/signature.cc @@ -634,7 +634,7 @@ absl::StatusOr ParseTypeSpec(std::string_view signature) { absl::StatusOr ParseType(std::string_view signature, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& pool) { CEL_ASSIGN_OR_RETURN(auto type_spec, ParseTypeSpec(signature)); - return cel::ConvertTypeSpecToType(type_spec, arena, pool); + return cel::ConvertTypeSpecToType(type_spec, pool, arena); } } // namespace cel diff --git a/common/signature_test.cc b/common/signature_test.cc index ea51eb566..e157d5be0 100644 --- a/common/signature_test.cc +++ b/common/signature_test.cc @@ -85,7 +85,7 @@ TEST_P(TypeSignatureTest, TypeSignature) { EXPECT_THAT(signature, IsOkAndHolds(param.expected_signature)); absl::StatusOr type = ConvertTypeSpecToType( - param.type, GetTestArena(), *GetTestingDescriptorPool()); + param.type, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(type, ::absl_testing::IsOk()); EXPECT_THAT(MakeTypeSignature(*type), IsOkAndHolds(param.expected_signature)); @@ -285,9 +285,10 @@ TEST_P(TypeSignatureTest, ParseTypeCheck) { auto parsed = ParseType(param.expected_signature, GetTestArena(), *GetTestingDescriptorPool()); ASSERT_THAT(parsed, ::absl_testing::IsOk()); - ASSERT_OK_AND_ASSIGN(auto expected_type, - ConvertTypeSpecToType(param.type, GetTestArena(), - *GetTestingDescriptorPool())); + ASSERT_OK_AND_ASSIGN( + auto expected_type, + ConvertTypeSpecToType(param.type, *GetTestingDescriptorPool(), + GetTestArena())); VerifyTypesEqual(*parsed, expected_type); } } diff --git a/common/type_spec_resolver.cc b/common/type_spec_resolver.cc index 90c9930a8..c3aa3d5a2 100644 --- a/common/type_spec_resolver.cc +++ b/common/type_spec_resolver.cc @@ -33,8 +33,8 @@ namespace cel { absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, - google::protobuf::Arena* arena, - const google::protobuf::DescriptorPool& pool) { + const google::protobuf::DescriptorPool& pool, + google::protobuf::Arena* arena) { if (type_spec.has_null()) return Type(NullType{}); if (type_spec.has_dyn()) return Type(DynType{}); @@ -94,7 +94,7 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, if (type_spec.list_type().elem_type().is_specified()) { CEL_ASSIGN_OR_RETURN( elem_type, ConvertTypeSpecToType(type_spec.list_type().elem_type(), - arena, pool)); + pool, arena)); } return Type(ListType(arena, elem_type)); } @@ -104,14 +104,14 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, if (type_spec.map_type().key_type().is_specified()) { CEL_ASSIGN_OR_RETURN( key_type, - ConvertTypeSpecToType(type_spec.map_type().key_type(), arena, pool)); + ConvertTypeSpecToType(type_spec.map_type().key_type(), pool, arena)); } Type value_type; if (type_spec.map_type().value_type().is_specified()) { CEL_ASSIGN_OR_RETURN( value_type, ConvertTypeSpecToType(type_spec.map_type().value_type(), - arena, pool)); + pool, arena)); } return Type(MapType(arena, key_type, value_type)); } @@ -122,13 +122,13 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, if (func_spec.result_type().is_specified()) { CEL_ASSIGN_OR_RETURN( result_type, - ConvertTypeSpecToType(func_spec.result_type(), arena, pool)); + ConvertTypeSpecToType(func_spec.result_type(), pool, arena)); } std::vector arg_types; arg_types.reserve(func_spec.arg_types().size()); for (const auto& arg_spec : func_spec.arg_types()) { CEL_ASSIGN_OR_RETURN(auto arg_type, - ConvertTypeSpecToType(arg_spec, arena, pool)); + ConvertTypeSpecToType(arg_spec, pool, arena)); arg_types.push_back(std::move(arg_type)); } return Type(FunctionType(arena, result_type, arg_types)); @@ -178,7 +178,7 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, std::vector params; for (const auto& param_spec : type_spec.abstract_type().parameter_types()) { CEL_ASSIGN_OR_RETURN(auto param, - ConvertTypeSpecToType(param_spec, arena, pool)); + ConvertTypeSpecToType(param_spec, pool, arena)); params.push_back(std::move(param)); } auto* allocated_name = google::protobuf::Arena::Create(arena, name); @@ -187,7 +187,7 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, if (type_spec.has_type()) { CEL_ASSIGN_OR_RETURN(auto contained_type, - ConvertTypeSpecToType(type_spec.type(), arena, pool)); + ConvertTypeSpecToType(type_spec.type(), pool, arena)); return Type(TypeType(arena, contained_type)); } diff --git a/common/type_spec_resolver.h b/common/type_spec_resolver.h index edbfa3bde..2cd860f02 100644 --- a/common/type_spec_resolver.h +++ b/common/type_spec_resolver.h @@ -29,8 +29,8 @@ namespace cel { // properties of the type when used in CEL. Returns a status with code // `InvalidArgument` if the input cannot be resolved to a type. absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, - google::protobuf::Arena* arena, - const google::protobuf::DescriptorPool& pool); + const google::protobuf::DescriptorPool& pool, + google::protobuf::Arena* arena); // Resolves a `cel::Type` to a `cel::TypeSpec`. absl::StatusOr ConvertTypeToTypeSpec(const Type& type); diff --git a/common/type_spec_resolver_test.cc b/common/type_spec_resolver_test.cc index 1cda7280f..dbde63e6d 100644 --- a/common/type_spec_resolver_test.cc +++ b/common/type_spec_resolver_test.cc @@ -49,7 +49,7 @@ google::protobuf::Arena* GetTestArena() { TEST(TypeSpecResolverTest, NullTypeSpec) { TypeSpec spec(NullTypeSpec{}); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsNull()); } @@ -57,7 +57,7 @@ TEST(TypeSpecResolverTest, NullTypeSpec) { TEST(TypeSpecResolverTest, DynTypeSpec) { TypeSpec spec(DynTypeSpec{}); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsDyn()); } @@ -66,8 +66,9 @@ using ConversionTest = testing::TestWithParam>; TEST_P(ConversionTest, TestTypeSpecConversion) { ASSERT_OK_AND_ASSIGN( - auto t, ConvertTypeSpecToType(std::get<0>(GetParam()), GetTestArena(), - *GetTestingDescriptorPool())); + auto t, + ConvertTypeSpecToType(std::get<0>(GetParam()), + *GetTestingDescriptorPool(), GetTestArena())); EXPECT_EQ(t.kind(), std::get<1>(GetParam())); EXPECT_THAT(ConvertTypeToTypeSpec(t), IsOkAndHolds(std::get<0>(GetParam()))); } @@ -103,7 +104,7 @@ TEST(TypeSpecResolverTest, ListTypeConversion) { auto elem = std::make_unique(PrimitiveType::kInt64); TypeSpec spec(ListTypeSpec(std::move(elem))); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsList()); EXPECT_TRUE(t->GetList().element().IsInt()); @@ -116,7 +117,7 @@ TEST(TypeSpecResolverTest, MapTypeConversion) { auto val = std::make_unique(PrimitiveType::kBytes); TypeSpec spec(MapTypeSpec(std::move(key), std::move(val))); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsMap()); EXPECT_TRUE(t->GetMap().key().IsString()); @@ -131,7 +132,7 @@ TEST(TypeSpecResolverTest, FunctionTypeConversion) { args.push_back(TypeSpec(PrimitiveType::kString)); TypeSpec spec(FunctionTypeSpec(std::move(result), std::move(args))); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsFunction()); EXPECT_EQ(t->GetFunction().args().size(), 1); @@ -143,7 +144,7 @@ TEST(TypeSpecResolverTest, FunctionTypeConversion) { TEST(TypeSpecResolverTest, TypeParamConversion) { TypeSpec spec(ParamTypeSpec("T")); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsTypeParam()); EXPECT_EQ(t->GetTypeParam().name(), "T"); @@ -155,7 +156,7 @@ TEST(TypeSpecResolverTest, MessageTypeConversion) { TypeSpec spec( AbstractType("cel.expr.conformance.proto3.TestAllTypes", /*params=*/{})); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsMessage()); EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes"); @@ -171,7 +172,7 @@ TEST(TypeSpecResolverTest, MessageTypeWithParamsError) { TypeSpec spec(AbstractType("cel.expr.conformance.proto3.TestAllTypes", std::move(params))); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("cannot have type parameters"))); } @@ -181,7 +182,7 @@ TEST(TypeSpecResolverTest, UnresolvedAbstractTypeFallbackToOpaque) { params.push_back(TypeSpec(PrimitiveType::kInt64)); TypeSpec spec(AbstractType("my.custom.OpaqueType", std::move(params))); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsOpaque()); EXPECT_EQ(t->name(), "my.custom.OpaqueType"); @@ -196,7 +197,7 @@ TEST(TypeSpecResolverTest, OptionalType) { params.push_back(TypeSpec(PrimitiveType::kInt64)); TypeSpec spec(AbstractType("optional_type", std::move(params))); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsOpaque()); EXPECT_EQ(t->name(), "optional_type"); @@ -211,7 +212,7 @@ TEST(TypeSpecResolverTest, TypeTypeConversion) { auto nested = std::make_unique(PrimitiveType::kInt64); TypeSpec spec(std::move(nested)); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsType()); EXPECT_TRUE(t->GetType().GetType().IsInt()); @@ -222,7 +223,7 @@ TEST(TypeSpecResolverTest, TypeTypeConversion) { TEST(TypeSpecResolverTest, ErrorTypeConversion) { TypeSpec spec(ErrorTypeSpec::kValue); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsError()); ASSERT_OK_AND_ASSIGN(auto spec2, ConvertTypeToTypeSpec(*t)); @@ -232,7 +233,7 @@ TEST(TypeSpecResolverTest, ErrorTypeConversion) { TEST(TypeSpecResolverTest, MessageTypeSpecConversion) { TypeSpec spec(MessageTypeSpec("cel.expr.conformance.proto3.TestAllTypes")); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsMessage()); EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes"); @@ -243,7 +244,7 @@ TEST(TypeSpecResolverTest, MessageTypeSpecConversion) { TEST(TypeSpecResolverTest, MessageTypeSpecNotFoundError) { TypeSpec spec(MessageTypeSpec("cel.expr.conformance.proto3.NonExistentType")); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("not found in descriptor pool"))); } @@ -252,7 +253,7 @@ TEST(TypeSpecResolverTest, EnumTypeConversion) { TypeSpec spec(AbstractType( "cel.expr.conformance.proto3.TestAllTypes.NestedEnum", /*params=*/{})); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); ASSERT_THAT(t, IsOk()); EXPECT_TRUE(t->IsEnum()); EXPECT_EQ(t->name(), "cel.expr.conformance.proto3.TestAllTypes.NestedEnum"); @@ -267,7 +268,7 @@ TEST(TypeSpecResolverTest, EnumTypeWithParamsError) { AbstractType("cel.expr.conformance.proto3.TestAllTypes.NestedEnum", std::move(params))); auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("cannot have type parameters"))); } @@ -275,7 +276,7 @@ TEST(TypeSpecResolverTest, EnumTypeWithParamsError) { TEST(TypeSpecResolverTest, UnknownTypeSpecKindError) { TypeSpec spec; auto t = - ConvertTypeSpecToType(spec, GetTestArena(), *GetTestingDescriptorPool()); + ConvertTypeSpecToType(spec, *GetTestingDescriptorPool(), GetTestArena()); EXPECT_THAT(t, StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Unknown TypeSpec kind"))); } From 4bf59ba35d1c3666a314230f2438f1eff4bfabed Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 20 Jul 2026 13:38:57 -0700 Subject: [PATCH 127/350] Update type_spec_resolver to support TypeIntrospector. Needed to support looking up types at runtime. PiperOrigin-RevId: 951027622 --- checker/internal/BUILD | 35 +------ checker/internal/type_check_env.h | 2 +- common/BUILD | 35 +++++++ .../descriptor_pool_type_introspector.cc | 8 +- .../descriptor_pool_type_introspector.h | 13 +-- .../descriptor_pool_type_introspector_test.cc | 8 +- common/type_spec_resolver.cc | 95 ++++++++++++------- common/type_spec_resolver.h | 6 ++ common/type_spec_resolver_test.cc | 57 +++++++++++ 9 files changed, 174 insertions(+), 85 deletions(-) rename {checker/internal => common}/descriptor_pool_type_introspector.cc (98%) rename {checker/internal => common}/descriptor_pool_type_introspector.h (91%) rename {checker/internal => common}/descriptor_pool_type_introspector_test.cc (97%) diff --git a/checker/internal/BUILD b/checker/internal/BUILD index 20c476db2..5e1bdbdcd 100644 --- a/checker/internal/BUILD +++ b/checker/internal/BUILD @@ -65,12 +65,12 @@ cc_library( srcs = ["type_check_env.cc"], hdrs = ["type_check_env.h"], deps = [ - ":descriptor_pool_type_introspector", ":proto_type_mask", ":proto_type_mask_registry", "//common:constant", "//common:container", "//common:decl", + "//common:descriptor_pool_type_introspector", "//common:type", "//internal:status_macros", "@com_google_absl//absl/base:core_headers", @@ -275,39 +275,6 @@ cc_test( ], ) -cc_library( - name = "descriptor_pool_type_introspector", - srcs = ["descriptor_pool_type_introspector.cc"], - hdrs = ["descriptor_pool_type_introspector.h"], - deps = [ - "//common:type", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings:string_view", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", - "@com_google_protobuf//:protobuf", - ], -) - -cc_test( - name = "descriptor_pool_type_introspector_test", - srcs = ["descriptor_pool_type_introspector_test.cc"], - deps = [ - ":descriptor_pool_type_introspector", - "//common:type", - "//internal:testing", - "//internal:testing_descriptor_pool", - "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", - ], -) - cc_library( name = "field_path", srcs = ["field_path.cc"], diff --git a/checker/internal/type_check_env.h b/checker/internal/type_check_env.h index 00fea0ba3..0fda608ab 100644 --- a/checker/internal/type_check_env.h +++ b/checker/internal/type_check_env.h @@ -30,12 +30,12 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" -#include "checker/internal/descriptor_pool_type_introspector.h" #include "checker/internal/proto_type_mask.h" #include "checker/internal/proto_type_mask_registry.h" #include "common/constant.h" #include "common/container.h" #include "common/decl.h" +#include "common/descriptor_pool_type_introspector.h" #include "common/type.h" #include "common/type_introspector.h" #include "internal/status_macros.h" diff --git a/common/BUILD b/common/BUILD index 0426c0827..a6b57ab31 100644 --- a/common/BUILD +++ b/common/BUILD @@ -52,12 +52,14 @@ cc_library( hdrs = ["type_spec_resolver.h"], deps = [ ":ast", + ":descriptor_pool_type_introspector", ":type", ":type_kind", "//internal:status_macros", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", ], ) @@ -79,6 +81,39 @@ cc_test( ], ) +cc_library( + name = "descriptor_pool_type_introspector", + srcs = ["descriptor_pool_type_introspector.cc"], + hdrs = ["descriptor_pool_type_introspector.h"], + deps = [ + ":type", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "descriptor_pool_type_introspector_test", + srcs = ["descriptor_pool_type_introspector_test.cc"], + deps = [ + ":descriptor_pool_type_introspector", + ":type", + "//internal:testing", + "//internal:testing_descriptor_pool", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/types:optional", + ], +) + cc_library( name = "signature", srcs = ["signature.cc"], diff --git a/checker/internal/descriptor_pool_type_introspector.cc b/common/descriptor_pool_type_introspector.cc similarity index 98% rename from checker/internal/descriptor_pool_type_introspector.cc rename to common/descriptor_pool_type_introspector.cc index 733e4a3cb..fa9ed2a0e 100644 --- a/checker/internal/descriptor_pool_type_introspector.cc +++ b/common/descriptor_pool_type_introspector.cc @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "checker/internal/descriptor_pool_type_introspector.h" +#include "common/descriptor_pool_type_introspector.h" #include +#include #include #include @@ -24,13 +25,12 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "common/type.h" #include "common/type_introspector.h" #include "google/protobuf/descriptor.h" -namespace cel::checker_internal { +namespace cel { namespace { // Standard implementation for field lookups. @@ -242,4 +242,4 @@ DescriptorPoolTypeIntrospector::CreateFieldTable( return result; } -} // namespace cel::checker_internal +} // namespace cel diff --git a/checker/internal/descriptor_pool_type_introspector.h b/common/descriptor_pool_type_introspector.h similarity index 91% rename from checker/internal/descriptor_pool_type_introspector.h rename to common/descriptor_pool_type_introspector.h index 8a970ea00..e7d6091d2 100644 --- a/checker/internal/descriptor_pool_type_introspector.h +++ b/common/descriptor_pool_type_introspector.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ -#define THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ +#ifndef THIRD_PARTY_CEL_CPP_COMMON_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ #include #include @@ -30,14 +30,15 @@ #include "common/type_introspector.h" #include "google/protobuf/descriptor.h" -namespace cel::checker_internal { +namespace cel { // Implementation of `TypeIntrospector` that uses a `google::protobuf::DescriptorPool`. // // This is used by the type checker to resolve protobuf types and their fields // and apply any options like using JSON names. // -// Neither copyable nor movable. Should be managed by a TypeCheckEnv. +// Neither copyable nor movable. Should be managed by a TypeCheckEnv or a +// runtime Environment. class DescriptorPoolTypeIntrospector : public TypeIntrospector { public: struct Field { @@ -100,6 +101,6 @@ class DescriptorPoolTypeIntrospector : public TypeIntrospector { const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; }; -} // namespace cel::checker_internal +} // namespace cel -#endif // THIRD_PARTY_CEL_CPP_CHECKER_INTERNAL_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ +#endif // THIRD_PARTY_CEL_CPP_COMMON_DESCRIPTOR_POOL_TYPE_INTROSPECTOR_H_ diff --git a/checker/internal/descriptor_pool_type_introspector_test.cc b/common/descriptor_pool_type_introspector_test.cc similarity index 97% rename from checker/internal/descriptor_pool_type_introspector_test.cc rename to common/descriptor_pool_type_introspector_test.cc index db766b347..94acd317f 100644 --- a/checker/internal/descriptor_pool_type_introspector_test.cc +++ b/common/descriptor_pool_type_introspector_test.cc @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "checker/internal/descriptor_pool_type_introspector.h" +#include "common/descriptor_pool_type_introspector.h" +#include #include #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "common/type.h" #include "common/type_introspector.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" -namespace cel::checker_internal { +namespace cel { namespace { using ::absl_testing::IsOkAndHolds; @@ -172,4 +172,4 @@ TEST(DescriptorPoolTypeIntrospectorTest, ListFieldsForStructTypeNotFound) { } } // namespace -} // namespace cel::checker_internal +} // namespace cel diff --git a/common/type_spec_resolver.cc b/common/type_spec_resolver.cc index c3aa3d5a2..aab099255 100644 --- a/common/type_spec_resolver.cc +++ b/common/type_spec_resolver.cc @@ -23,18 +23,36 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include "common/ast.h" +#include "common/descriptor_pool_type_introspector.h" #include "common/type.h" +#include "common/type_introspector.h" #include "common/type_kind.h" #include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" namespace cel { +namespace { -absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, - const google::protobuf::DescriptorPool& pool, - google::protobuf::Arena* arena) { +absl::StatusOr> ResolveNamedType( + absl::string_view name, const TypeIntrospector& type_introspector) { + absl::optional type = FindWellKnownType(name); + if (type.has_value()) { + return type; + } + return type_introspector.FindType(name); +} + +bool TypeAcceptsParameters(const Type& type) { + return type.IsOpaque() || type.IsType() || type.IsList() || type.IsMap(); +} +} // namespace + +absl::StatusOr ConvertTypeSpecToType( + const TypeSpec& type_spec, const TypeIntrospector& type_introspector, + google::protobuf::Arena* arena) { if (type_spec.has_null()) return Type(NullType{}); if (type_spec.has_dyn()) return Type(DynType{}); @@ -94,7 +112,7 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, if (type_spec.list_type().elem_type().is_specified()) { CEL_ASSIGN_OR_RETURN( elem_type, ConvertTypeSpecToType(type_spec.list_type().elem_type(), - pool, arena)); + type_introspector, arena)); } return Type(ListType(arena, elem_type)); } @@ -103,15 +121,15 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, Type key_type; if (type_spec.map_type().key_type().is_specified()) { CEL_ASSIGN_OR_RETURN( - key_type, - ConvertTypeSpecToType(type_spec.map_type().key_type(), pool, arena)); + key_type, ConvertTypeSpecToType(type_spec.map_type().key_type(), + type_introspector, arena)); } Type value_type; if (type_spec.map_type().value_type().is_specified()) { CEL_ASSIGN_OR_RETURN( value_type, ConvertTypeSpecToType(type_spec.map_type().value_type(), - pool, arena)); + type_introspector, arena)); } return Type(MapType(arena, key_type, value_type)); } @@ -120,15 +138,16 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, const auto& func_spec = type_spec.function(); Type result_type; if (func_spec.result_type().is_specified()) { - CEL_ASSIGN_OR_RETURN( - result_type, - ConvertTypeSpecToType(func_spec.result_type(), pool, arena)); + CEL_ASSIGN_OR_RETURN(result_type, + ConvertTypeSpecToType(func_spec.result_type(), + type_introspector, arena)); } std::vector arg_types; arg_types.reserve(func_spec.arg_types().size()); for (const auto& arg_spec : func_spec.arg_types()) { - CEL_ASSIGN_OR_RETURN(auto arg_type, - ConvertTypeSpecToType(arg_spec, pool, arena)); + CEL_ASSIGN_OR_RETURN( + auto arg_type, + ConvertTypeSpecToType(arg_spec, type_introspector, arena)); arg_types.push_back(std::move(arg_type)); } return Type(FunctionType(arena, result_type, arg_types)); @@ -142,43 +161,36 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, if (type_spec.has_message_type()) { const std::string& name = type_spec.message_type().type(); - const google::protobuf::Descriptor* descriptor = pool.FindMessageTypeByName(name); - if (descriptor == nullptr) { + CEL_ASSIGN_OR_RETURN(absl::optional type, + ResolveNamedType(name, type_introspector)); + if (!type.has_value()) { return absl::InvalidArgumentError(absl::StrCat( "Message type '", name, "' not found in descriptor pool")); } - return Type::Message(descriptor); + return *type; } if (type_spec.has_abstract_type()) { const std::string& name = type_spec.abstract_type().name(); - // Check if it's a message type in the pool - const google::protobuf::Descriptor* descriptor = pool.FindMessageTypeByName(name); - if (descriptor != nullptr) { - if (!type_spec.abstract_type().parameter_types().empty()) { - return absl::InvalidArgumentError(absl::StrCat( - "Message type '", name, "' cannot have type parameters")); - } - return Type::Message(descriptor); - } - - // Check if it's an enum type in the pool - const google::protobuf::EnumDescriptor* enum_descriptor = - pool.FindEnumTypeByName(name); - if (enum_descriptor != nullptr) { - if (!type_spec.abstract_type().parameter_types().empty()) { + CEL_ASSIGN_OR_RETURN(absl::optional type, + ResolveNamedType(name, type_introspector)); + if (type.has_value()) { + if (!TypeAcceptsParameters(*type) && + !type_spec.abstract_type().parameter_types().empty()) { return absl::InvalidArgumentError( - absl::StrCat("Enum type '", name, "' cannot have type parameters")); + absl::StrCat("Type '", name, "' cannot have type parameters")); } - return Type::Enum(enum_descriptor); + return *type; } // Otherwise fallback to OpaqueType std::vector params; + params.reserve(type_spec.abstract_type().parameter_types().size()); for (const auto& param_spec : type_spec.abstract_type().parameter_types()) { - CEL_ASSIGN_OR_RETURN(auto param, - ConvertTypeSpecToType(param_spec, pool, arena)); + CEL_ASSIGN_OR_RETURN( + auto param, + ConvertTypeSpecToType(param_spec, type_introspector, arena)); params.push_back(std::move(param)); } auto* allocated_name = google::protobuf::Arena::Create(arena, name); @@ -186,8 +198,9 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, } if (type_spec.has_type()) { - CEL_ASSIGN_OR_RETURN(auto contained_type, - ConvertTypeSpecToType(type_spec.type(), pool, arena)); + CEL_ASSIGN_OR_RETURN( + auto contained_type, + ConvertTypeSpecToType(type_spec.type(), type_introspector, arena)); return Type(TypeType(arena, contained_type)); } @@ -198,6 +211,16 @@ absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, return absl::InvalidArgumentError("Unknown TypeSpec kind"); } +absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, + const google::protobuf::DescriptorPool& pool, + google::protobuf::Arena* arena) { + // In this configuration we will only ever reference objects backed by the + // given DescriptorPool or global constants so it is safe for `introspector` + // to go out of scope, but this is not sound for arbitrary TypeIntrospectors. + DescriptorPoolTypeIntrospector introspector(&pool); + return ConvertTypeSpecToType(type_spec, introspector, arena); +} + absl::StatusOr ConvertTypeToTypeSpec(const Type& type) { switch (type.kind()) { case TypeKind::kNull: diff --git a/common/type_spec_resolver.h b/common/type_spec_resolver.h index 2cd860f02..17fbf27de 100644 --- a/common/type_spec_resolver.h +++ b/common/type_spec_resolver.h @@ -18,6 +18,7 @@ #include "absl/status/statusor.h" #include "common/ast.h" #include "common/type.h" +#include "common/type_introspector.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -28,6 +29,11 @@ namespace cel { // TypeSpec only specifies a type while Type provides support for inspecting // properties of the type when used in CEL. Returns a status with code // `InvalidArgument` if the input cannot be resolved to a type. +absl::StatusOr ConvertTypeSpecToType( + const TypeSpec& type_spec, const cel::TypeIntrospector& type_introspector, + google::protobuf::Arena* arena); + +// Convenience overload for type resolution using only a descriptor pool. absl::StatusOr ConvertTypeSpecToType(const TypeSpec& type_spec, const google::protobuf::DescriptorPool& pool, google::protobuf::Arena* arena); diff --git a/common/type_spec_resolver_test.cc b/common/type_spec_resolver_test.cc index dbde63e6d..056ba8f67 100644 --- a/common/type_spec_resolver_test.cc +++ b/common/type_spec_resolver_test.cc @@ -25,6 +25,7 @@ #include "absl/status/status_matchers.h" #include "common/ast.h" #include "common/type.h" +#include "common/type_introspector.h" #include "common/type_kind.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" @@ -273,6 +274,62 @@ TEST(TypeSpecResolverTest, EnumTypeWithParamsError) { HasSubstr("cannot have type parameters"))); } +TEST(TypeSpecResolverTest, WellKnownMessageTypeFiltering) { + TypeSpec message_spec(MessageTypeSpec("google.protobuf.Timestamp")); + ASSERT_OK_AND_ASSIGN( + auto message_type, + ConvertTypeSpecToType(message_spec, *GetTestingDescriptorPool(), + GetTestArena())); + EXPECT_TRUE(message_type.IsTimestamp()); + + TypeSpec abstract_spec( + AbstractType("google.protobuf.Duration", /*params=*/{})); + ASSERT_OK_AND_ASSIGN( + auto abstract_type, + ConvertTypeSpecToType(abstract_spec, *GetTestingDescriptorPool(), + GetTestArena())); + EXPECT_TRUE(abstract_type.IsDuration()); + + std::vector params; + params.push_back(TypeSpec(PrimitiveType::kInt64)); + TypeSpec abstract_with_params( + AbstractType("google.protobuf.Timestamp", std::move(params))); + EXPECT_THAT( + ConvertTypeSpecToType(abstract_with_params, *GetTestingDescriptorPool(), + GetTestArena()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("cannot have type parameters"))); +} + +TEST(TypeSpecResolverTest, WellKnownTypeIntrospector) { + WellKnownTypeIntrospector introspector; + + TypeSpec message_spec(MessageTypeSpec("google.protobuf.Timestamp")); + ASSERT_OK_AND_ASSIGN( + auto message_type, + ConvertTypeSpecToType(message_spec, introspector, GetTestArena())); + EXPECT_TRUE(message_type.IsTimestamp()); + + TypeSpec abstract_spec( + AbstractType("google.protobuf.Duration", /*params=*/{})); + ASSERT_OK_AND_ASSIGN( + auto abstract_type, + ConvertTypeSpecToType(abstract_spec, introspector, GetTestArena())); + EXPECT_TRUE(abstract_type.IsDuration()); + + TypeSpec primitive_spec(PrimitiveType::kInt64); + ASSERT_OK_AND_ASSIGN( + auto primitive_type, + ConvertTypeSpecToType(primitive_spec, introspector, GetTestArena())); + EXPECT_TRUE(primitive_type.IsInt()); + + TypeSpec custom_spec(MessageTypeSpec("custom.UnknownMessage")); + EXPECT_THAT( + ConvertTypeSpecToType(custom_spec, introspector, GetTestArena()), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Message type 'custom.UnknownMessage' not found"))); +} + TEST(TypeSpecResolverTest, UnknownTypeSpecKindError) { TypeSpec spec; auto t = From 53547225df3f531ddcb5dbff68d909ec48f77618 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 22 Jul 2026 14:13:33 -0700 Subject: [PATCH 128/350] Add interruptable ast traversal utility for proto ASTs (CheckedExpr). PiperOrigin-RevId: 952324593 --- common/ast_traverse.h | 2 -- eval/public/ast_traverse.cc | 42 +++++++++++++++++++++++++++++ eval/public/ast_traverse.h | 43 ++++++++++++++++++++++++++++++ eval/public/ast_traverse_test.cc | 45 ++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 2 deletions(-) diff --git a/common/ast_traverse.h b/common/ast_traverse.h index 004727e49..21272a898 100644 --- a/common/ast_traverse.h +++ b/common/ast_traverse.h @@ -43,8 +43,6 @@ struct TraversalOptions { // while(!traversal.IsDone()) { // traversal.Step(visitor); // } -// -// This class is thread-hostile and should only be used in synchronous code. class AstTraversal { public: static AstTraversal Create(const cel::Expr& ast ABSL_ATTRIBUTE_LIFETIME_BOUND, diff --git a/eval/public/ast_traverse.cc b/eval/public/ast_traverse.cc index c18b806b9..a31e1d37d 100644 --- a/eval/public/ast_traverse.cc +++ b/eval/public/ast_traverse.cc @@ -14,6 +14,7 @@ #include "eval/public/ast_traverse.h" +#include #include #include "cel/expr/syntax.pb.h" @@ -344,6 +345,47 @@ void PushDependencies(const StackRecord& record, std::stack& stack, } // namespace +namespace internal { +struct AstTraversalState { + std::stack stack; +}; +} // namespace internal + +AstTraversal AstTraversal::Create(const Expr* expr, + const SourceInfo* source_info, + TraversalOptions options) { + AstTraversal instance(options); + instance.state_ = std::make_unique(); + instance.state_->stack.push(StackRecord(expr, source_info)); + return instance; +} + +AstTraversal::AstTraversal(TraversalOptions options) : options_(options) {} + +AstTraversal::~AstTraversal() = default; + +bool AstTraversal::Step(AstVisitor* visitor) { + if (IsDone()) { + return false; + } + auto& stack = state_->stack; + StackRecord& record = stack.top(); + if (!record.visited) { + PreVisit(record, visitor); + PushDependencies(record, stack, options_); + record.visited = true; + } else { + PostVisit(record, visitor); + stack.pop(); + } + + return !stack.empty(); +} + +bool AstTraversal::IsDone() { + return state_ == nullptr || state_->stack.empty(); +} + void AstTraverse(const Expr* expr, const SourceInfo* source_info, AstVisitor* visitor, TraversalOptions options) { std::stack stack; diff --git a/eval/public/ast_traverse.h b/eval/public/ast_traverse.h index f81c6f47a..a330bd28d 100644 --- a/eval/public/ast_traverse.h +++ b/eval/public/ast_traverse.h @@ -17,17 +17,60 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_AST_TRAVERSE_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_AST_TRAVERSE_H_ +#include + #include "cel/expr/syntax.pb.h" #include "eval/public/ast_visitor.h" namespace google::api::expr::runtime { +namespace internal { +struct AstTraversalState; +} // namespace internal + struct TraversalOptions { bool use_comprehension_callbacks; TraversalOptions() : use_comprehension_callbacks(false) {} }; +// Helper class for managing the traversal of the AST. +// Allows caller to step through the traversal. +// +// Usage: +// +// AstTraversal traversal = AstTraversal::Create(expr, source_info); +// +// MyVisitor visitor(); +// while (!traversal.IsDone()) { +// traversal.Step(&visitor); +// } +class AstTraversal { + public: + static AstTraversal Create(const cel::expr::Expr* expr, + const cel::expr::SourceInfo* source_info, + TraversalOptions options = TraversalOptions()); + + ~AstTraversal(); + + AstTraversal(const AstTraversal&) = delete; + AstTraversal& operator=(const AstTraversal&) = delete; + AstTraversal(AstTraversal&&) = default; + AstTraversal& operator=(AstTraversal&&) = default; + + // Advances the traversal. Returns true if there is more work to do. This is a + // no-op if the traversal is done and IsDone() is true. + bool Step(AstVisitor* visitor); + + // Returns true if there is no work left to do. + bool IsDone(); + + private: + explicit AstTraversal(TraversalOptions options); + TraversalOptions options_; + std::unique_ptr state_; +}; + // Traverses the AST representation in an expr proto. // // expr: root node of the tree. diff --git a/eval/public/ast_traverse_test.cc b/eval/public/ast_traverse_test.cc index ca6d81b72..dfc4a7700 100644 --- a/eval/public/ast_traverse_test.cc +++ b/eval/public/ast_traverse_test.cc @@ -465,6 +465,51 @@ TEST(AstCrawlerTest, CheckExprHandlers) { AstTraverse(&expr, &source_info, &handler); } +TEST(AstTraversal, Interrupt) { + SourceInfo source_info; + MockAstVisitor handler; + + Expr expr; + auto* select_expr = expr.mutable_select_expr(); + auto* operand = select_expr->mutable_operand(); + auto* ident_expr = operand->mutable_ident_expr(); + + testing::InSequence seq; + + auto traversal = AstTraversal::Create(&expr, &source_info); + + EXPECT_CALL(handler, PreVisitExpr(_, _)).Times(2); + + EXPECT_CALL(handler, PostVisitIdent(ident_expr, operand, _)).Times(1); + EXPECT_CALL(handler, PostVisitSelect(select_expr, &expr, _)).Times(0); + + EXPECT_TRUE(traversal.Step(&handler)); + EXPECT_TRUE(traversal.Step(&handler)); + EXPECT_TRUE(traversal.Step(&handler)); + + EXPECT_FALSE(traversal.IsDone()); +} + +TEST(AstTraversal, NoInterrupt) { + SourceInfo source_info; + MockAstVisitor handler; + + Expr expr; + auto* select_expr = expr.mutable_select_expr(); + auto* operand = select_expr->mutable_operand(); + auto* ident_expr = operand->mutable_ident_expr(); + + testing::InSequence seq; + + auto traversal = AstTraversal::Create(&expr, &source_info); + + EXPECT_CALL(handler, PostVisitIdent(ident_expr, operand, _)).Times(1); + EXPECT_CALL(handler, PostVisitSelect(select_expr, &expr, _)).Times(1); + + while (traversal.Step(&handler)) continue; + EXPECT_TRUE(traversal.IsDone()); +} + } // namespace } // namespace google::api::expr::runtime From b09d20964f64d8a5882a80640f5b6915d02663b6 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 22 Jul 2026 16:29:46 -0700 Subject: [PATCH 129/350] Misc consistency/style fixes for Struct types: - add helper for exposing operations on wrapped legacy value messages - avoid a string copy in AttibuteTrail::Step when empty - add accessors for ParsedMessage::message ::descriptor - update legacy type provider to report typed fields for normal (no custom accesor impl) protos - test helper for working with dynamic messages in the hermetic test environment No functional changes PiperOrigin-RevId: 952396769 --- common/descriptor_pool_type_introspector.cc | 3 +- common/legacy_value.cc | 47 ++++++++++--- common/legacy_value.h | 15 ++++ common/type.cc | 5 ++ common/type.h | 1 + common/types/message_type.h | 13 +++- common/value.cc | 1 + common/values/parsed_message_value.h | 24 +++---- eval/eval/attribute_trail.h | 1 + eval/public/structs/BUILD | 6 ++ eval/public/structs/legacy_type_info_apis.h | 5 ++ eval/public/structs/legacy_type_provider.cc | 42 +++++++---- .../structs/legacy_type_provider_test.cc | 28 ++++++++ .../structs/proto_message_type_adapter.cc | 44 ++++++------ .../structs/proto_message_type_adapter.h | 8 +++ .../proto_message_type_adapter_test.cc | 55 +++++++++++++++ .../protobuf_descriptor_type_provider_test.cc | 69 ++++++++++++++++++- runtime/internal/BUILD | 1 + runtime/internal/runtime_env_testing.h | 31 +++++++++ 19 files changed, 335 insertions(+), 64 deletions(-) diff --git a/common/descriptor_pool_type_introspector.cc b/common/descriptor_pool_type_introspector.cc index fa9ed2a0e..980b2d25d 100644 --- a/common/descriptor_pool_type_introspector.cc +++ b/common/descriptor_pool_type_introspector.cc @@ -44,8 +44,7 @@ FindStructTypeFieldByNameDirectly( if (descriptor == nullptr) { return std::nullopt; } - const google::protobuf::FieldDescriptor* absl_nullable field = - descriptor->FindFieldByName(name); + const google::protobuf::FieldDescriptor* field = descriptor->FindFieldByName(name); if (field != nullptr) { return StructTypeField(MessageTypeField(field)); } diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 7fbf16732..50de9a0f8 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -71,14 +71,15 @@ namespace cel { namespace { -using google::api::expr::runtime::CelList; -using google::api::expr::runtime::CelMap; -using google::api::expr::runtime::CelValue; -using google::api::expr::runtime::FieldBackedListImpl; -using google::api::expr::runtime::FieldBackedMapImpl; -using google::api::expr::runtime::GetGenericProtoTypeInfoInstance; -using google::api::expr::runtime::LegacyTypeInfoApis; -using google::api::expr::runtime::MessageWrapper; +using ::google::api::expr::runtime::CelList; +using ::google::api::expr::runtime::CelMap; +using ::google::api::expr::runtime::CelValue; +using ::google::api::expr::runtime::CreateCelValueFromField; +using ::google::api::expr::runtime::FieldBackedListImpl; +using ::google::api::expr::runtime::FieldBackedMapImpl; +using ::google::api::expr::runtime::GetGenericProtoTypeInfoInstance; +using ::google::api::expr::runtime::LegacyTypeInfoApis; +using ::google::api::expr::runtime::MessageWrapper; using ::google::api::expr::runtime::internal::MaybeWrapValueToMessage; absl::Status InvalidMapKeyTypeError(ValueKind kind) { @@ -1288,6 +1289,36 @@ TypeValue CreateTypeValueFromView(google::protobuf::Arena* arena, return TypeValue(common_internal::LegacyRuntimeType(input)); } +const google::protobuf::Message* absl_nullable GetLegacyMessage(const Value& value) { + if (!common_internal::IsLegacyStructValue(value)) { + return nullptr; + } + + auto legacy = common_internal::GetLegacyStructValue(value); + const auto* legacy_type_info = legacy.legacy_type_info(); + if (legacy_type_info == nullptr) { + return nullptr; + } + if (legacy_type_info != &GetGenericProtoTypeInfoInstance()) { + return nullptr; + } + if (IsWellKnownMessageType(legacy.message_ptr()->GetDescriptor())) { + return nullptr; + } + return legacy.message_ptr(); +} + +absl::Status WrapLegacyMessageField( + const google::protobuf::Message* absl_nonnull message, + const google::protobuf::FieldDescriptor* absl_nonnull field_descriptor, + ProtoWrapperTypeOptions unboxing_option, google::protobuf::Arena* arena, + Value* absl_nonnull out) { + CEL_ASSIGN_OR_RETURN(CelValue result, + CreateCelValueFromField(message, field_descriptor, + unboxing_option, arena)); + return ModernValue(arena, result, *out); +} + } // namespace interop_internal } // namespace cel diff --git a/common/legacy_value.h b/common/legacy_value.h index 7e703cea1..8d0392f7a 100644 --- a/common/legacy_value.h +++ b/common/legacy_value.h @@ -27,7 +27,9 @@ #include "common/value.h" #include "eval/public/cel_value.h" #include "internal/status_macros.h" +#include "runtime/runtime_options.h" #include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" namespace cel { @@ -59,6 +61,19 @@ google::api::expr::runtime::CelValue UnsafeLegacyValue( namespace cel::interop_internal { +// Returns the underlying `google::protobuf::Message` of a `cel::Value` if it is a legacy +// message with the default type info, or `nullptr` otherwise. +const google::protobuf::Message* absl_nullable GetLegacyMessage(const Value& value); + +// Access a field on a legacy message value, writing the result to `out`. +// Prefers wrapping legacy values instead of using the modern value +// representation. +absl::Status WrapLegacyMessageField( + const google::protobuf::Message* absl_nonnull message, + const google::protobuf::FieldDescriptor* absl_nonnull field_descriptor, + ProtoWrapperTypeOptions unboxing_option, google::protobuf::Arena* arena, + Value* absl_nonnull out); + absl::StatusOr FromLegacyValue( google::protobuf::Arena* arena, const google::api::expr::runtime::CelValue& legacy_value, diff --git a/common/type.cc b/common/type.cc index 9ea85954c..2d3469540 100644 --- a/common/type.cc +++ b/common/type.cc @@ -606,6 +606,11 @@ absl::optional StructTypeField::AsMessage() const { return std::nullopt; } +MessageTypeField StructTypeField::GetMessage() const { + ABSL_DCHECK(IsMessage()); + return absl::get(variant_); +} + StructTypeField::operator MessageTypeField() const { ABSL_DCHECK(IsMessage()); return absl::get(variant_); diff --git a/common/type.h b/common/type.h index c8851dd4e..e60452bfd 100644 --- a/common/type.h +++ b/common/type.h @@ -1140,6 +1140,7 @@ class StructTypeField final { } absl::optional AsMessage() const; + MessageTypeField GetMessage() const; explicit operator MessageTypeField() const; diff --git a/common/types/message_type.h b/common/types/message_type.h index 782af87aa..eea811971 100644 --- a/common/types/message_type.h +++ b/common/types/message_type.h @@ -87,6 +87,11 @@ class MessageType final { return descriptor_; } + const google::protobuf::Descriptor* absl_nonnull descriptor() const { + ABSL_DCHECK(*this); + return descriptor_; + } + explicit operator bool() const { return descriptor_ != nullptr; } private: @@ -166,8 +171,12 @@ class MessageTypeField final { return *descriptor_; } - const google::protobuf::FieldDescriptor* absl_nonnull operator->() const - ABSL_ATTRIBUTE_LIFETIME_BOUND { + const google::protobuf::FieldDescriptor* absl_nonnull operator->() const { + ABSL_DCHECK(*this); + return descriptor_; + } + + const google::protobuf::FieldDescriptor* absl_nonnull descriptor() const { ABSL_DCHECK(*this); return descriptor_; } diff --git a/common/value.cc b/common/value.cc index f13ee958e..6284626da 100644 --- a/common/value.cc +++ b/common/value.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/common/values/parsed_message_value.h b/common/values/parsed_message_value.h index f3d1f7b40..2e356d3e8 100644 --- a/common/values/parsed_message_value.h +++ b/common/values/parsed_message_value.h @@ -99,10 +99,8 @@ class ParsedMessageValue final return *value_; } - const google::protobuf::Message* absl_nonnull operator->() const - ABSL_ATTRIBUTE_LIFETIME_BOUND { - return value_; - } + const google::protobuf::Message* absl_nonnull operator->() const { return value_; } + const google::protobuf::Message* absl_nonnull message() const { return value_; } bool IsZeroValue() const; @@ -175,6 +173,15 @@ class ParsedMessageValue final swap(lhs.arena_, rhs.arena_); } + absl::Status GetField( + const google::protobuf::FieldDescriptor* absl_nonnull field, + ProtoWrapperTypeOptions unboxing_options, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const; + + bool HasField(const google::protobuf::FieldDescriptor* absl_nonnull field) const; + private: friend std::pointer_traits; friend class StructValue; @@ -203,15 +210,6 @@ class ParsedMessageValue final return absl::OkStatus(); } - absl::Status GetField( - const google::protobuf::FieldDescriptor* absl_nonnull field, - ProtoWrapperTypeOptions unboxing_options, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const; - - bool HasField(const google::protobuf::FieldDescriptor* absl_nonnull field) const; - const google::protobuf::Message* absl_nonnull value_; // Arena that is attributed as owning the value. May be null to indicate that // the value is managed externally. diff --git a/eval/eval/attribute_trail.h b/eval/eval/attribute_trail.h index 576d0be34..838cc8875 100644 --- a/eval/eval/attribute_trail.h +++ b/eval/eval/attribute_trail.h @@ -47,6 +47,7 @@ class AttributeTrail { // Creates AttributeTrail with attribute path incremented by "qualifier". AttributeTrail Step(const std::string* qualifier) const { + if (empty()) return AttributeTrail(); return Step(cel::AttributeQualifier::OfString(*qualifier)); } diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index d722559e3..2c7d7ed16 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -325,6 +325,7 @@ cc_test( ":proto_message_type_adapter", "//base:attributes", "//common:value", + "//common:value_testing", "//eval/public:cel_value", "//eval/public:message_wrapper", "//eval/public/containers:container_backed_list_impl", @@ -365,10 +366,13 @@ cc_test( deps = [ ":legacy_type_info_apis", ":protobuf_descriptor_type_provider", + "//common:type", "//eval/public:cel_value", "//eval/public/testing:matchers", "//extensions/protobuf:memory_manager", "//internal:testing", + "@com_google_absl//absl/status:status_matchers", + "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:wrappers_cc_proto", ], @@ -414,7 +418,9 @@ cc_test( deps = [ ":legacy_type_info_apis", ":legacy_type_provider", + "//common:type", "//internal:testing", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings:string_view", ], ) diff --git a/eval/public/structs/legacy_type_info_apis.h b/eval/public/structs/legacy_type_info_apis.h index 4f07470a1..e470ac566 100644 --- a/eval/public/structs/legacy_type_info_apis.h +++ b/eval/public/structs/legacy_type_info_apis.h @@ -61,6 +61,11 @@ class LegacyTypeInfoApis { virtual absl::string_view GetTypename( const MessageWrapper& wrapped_message) const = 0; + // Return a pointer to the descriptor for the wrapped message's type. + // + // Should only be defined for messages with standard behavior (i.e. normal + // duck-typed behavior of resolving fields by associated descriptor is + // correct). virtual const google::protobuf::Descriptor* absl_nullable GetDescriptor( const MessageWrapper& wrapped_message [[maybe_unused]]) const { return nullptr; diff --git a/eval/public/structs/legacy_type_provider.cc b/eval/public/structs/legacy_type_provider.cc index f8db92298..cd7079451 100644 --- a/eval/public/structs/legacy_type_provider.cc +++ b/eval/public/structs/legacy_type_provider.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include "absl/base/nullability.h" @@ -35,6 +36,7 @@ #include "extensions/protobuf/memory_manager.h" #include "internal/status_macros.h" #include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace google::api::expr::runtime { @@ -197,22 +199,34 @@ LegacyTypeProvider::FindStructTypeFieldByNameImpl( result.has_value()) { return result; } - if (auto type_info = ProvideLegacyTypeInfo(type); type_info.has_value()) { - if (auto field_desc = (*type_info)->FindFieldByName(name); - field_desc.has_value()) { - return cel::common_internal::BasicStructTypeField( - field_desc->name, field_desc->number, cel::DynType{}); - } else { - const auto* mutation_apis = - (*type_info)->GetMutationApis(MessageWrapper()); - if (mutation_apis == nullptr || !mutation_apis->DefinesField(name)) { - return std::nullopt; - } - return cel::common_internal::BasicStructTypeField(name, 0, - cel::DynType{}); + absl::optional type_info = + ProvideLegacyTypeInfo(type); + if (!type_info.has_value()) { + return std::nullopt; + } + if (const auto* descriptor = (*type_info)->GetDescriptor(MessageWrapper()); + descriptor != nullptr) { + // If it's a normal proto, just use the descriptor to find the field. + // Allows us to get the same optimizations as the modern value in most + // cases. + const google::protobuf::FieldDescriptor* field = descriptor->FindFieldByName(name); + if (field != nullptr) { + return cel::StructTypeField(cel::MessageTypeField(field)); } } - return std::nullopt; + + if (auto field_desc = (*type_info)->FindFieldByName(name); + field_desc.has_value()) { + return cel::common_internal::BasicStructTypeField( + field_desc->name, field_desc->number, cel::DynType{}); + } + + const auto* mutation_apis = (*type_info)->GetMutationApis(MessageWrapper()); + if (mutation_apis == nullptr || !mutation_apis->DefinesField(name)) { + return std::nullopt; + } + + return cel::common_internal::BasicStructTypeField(name, 0, cel::DynType{}); } } // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_provider_test.cc b/eval/public/structs/legacy_type_provider_test.cc index 8de2aba01..2da45e69d 100644 --- a/eval/public/structs/legacy_type_provider_test.cc +++ b/eval/public/structs/legacy_type_provider_test.cc @@ -17,13 +17,17 @@ #include #include +#include "absl/status/status_matchers.h" #include "absl/strings/string_view.h" +#include "common/type.h" #include "eval/public/structs/legacy_type_info_apis.h" #include "internal/testing.h" namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; + class LegacyTypeProviderTestEmpty : public LegacyTypeProvider { public: absl::optional ProvideLegacyType( @@ -46,6 +50,13 @@ class LegacyTypeInfoApisEmpty : public LegacyTypeInfoApis { const MessageWrapper& wrapped_message) const override { return nullptr; } + absl::optional FindFieldByName( + absl::string_view name) const override { + if (name == "field1") { + return FieldDescription{1, "field1"}; + } + return absl::nullopt; + } private: const std::string test_string_ = "test"; @@ -89,5 +100,22 @@ TEST(LegacyTypeProviderTest, NonEmptyTypeProviderProvidesSomeTypes) { EXPECT_EQ(provider.ProvideLegacyTypeInfo("other"), std::nullopt); } +TEST(LegacyTypeProviderTest, FindStructTypeFieldByName) { + LegacyTypeInfoApisEmpty test_type_info; + LegacyTypeProviderTestImpl provider(&test_type_info); + + ASSERT_OK_AND_ASSIGN(absl::optional field, + provider.FindStructTypeFieldByName("test", "field1")); + ASSERT_TRUE(field.has_value()); + EXPECT_EQ(field->name(), "field1"); + EXPECT_EQ(field->number(), 1); + EXPECT_EQ(field->GetType(), cel::DynType()); + + ASSERT_OK_AND_ASSIGN( + absl::optional not_found_field, + provider.FindStructTypeFieldByName("test", "unknown_field")); + EXPECT_FALSE(not_found_field.has_value()); +} + } // namespace } // namespace google::api::expr::runtime diff --git a/eval/public/structs/proto_message_type_adapter.cc b/eval/public/structs/proto_message_type_adapter.cc index 8c140c0c7..a8bb852fc 100644 --- a/eval/public/structs/proto_message_type_adapter.cc +++ b/eval/public/structs/proto_message_type_adapter.cc @@ -142,28 +142,6 @@ absl::StatusOr HasFieldImpl(const google::protobuf::Message* message, return CelFieldIsPresent(message, field_desc, reflection); } -absl::StatusOr CreateCelValueFromField( - const google::protobuf::Message* message, const google::protobuf::FieldDescriptor* field_desc, - ProtoWrapperTypeOptions unboxing_option, google::protobuf::Arena* arena) { - if (field_desc->is_map()) { - auto* map = google::protobuf::Arena::Create( - arena, message, field_desc, &MessageCelValueFactory, arena); - - return CelValue::CreateMap(map); - } - if (field_desc->is_repeated()) { - auto* list = google::protobuf::Arena::Create( - arena, message, field_desc, &MessageCelValueFactory, arena); - return CelValue::CreateList(list); - } - - CEL_ASSIGN_OR_RETURN( - CelValue result, - internal::CreateValueFromSingleField(message, field_desc, unboxing_option, - &MessageCelValueFactory, arena)); - return result; -} - // Shared implementation for GetField. // Handles list or map specific behavior before calling reflection helpers. absl::StatusOr GetFieldImpl(const google::protobuf::Message* message, @@ -441,6 +419,28 @@ CelValue MessageCelValueFactory(const google::protobuf::Message* message) { } // namespace +absl::StatusOr CreateCelValueFromField( + const google::protobuf::Message* message, const google::protobuf::FieldDescriptor* field_desc, + ProtoWrapperTypeOptions unboxing_option, google::protobuf::Arena* arena) { + if (field_desc->is_map()) { + auto* map = google::protobuf::Arena::Create( + arena, message, field_desc, &MessageCelValueFactory, arena); + + return CelValue::CreateMap(map); + } + if (field_desc->is_repeated()) { + auto* list = google::protobuf::Arena::Create( + arena, message, field_desc, &MessageCelValueFactory, arena); + return CelValue::CreateList(list); + } + + CEL_ASSIGN_OR_RETURN( + CelValue result, + internal::CreateValueFromSingleField(message, field_desc, unboxing_option, + &MessageCelValueFactory, arena)); + return result; +} + std::string ProtoMessageTypeAdapter::DebugString( const MessageWrapper& wrapped_message) const { if (!wrapped_message.HasFullProto() || diff --git a/eval/public/structs/proto_message_type_adapter.h b/eval/public/structs/proto_message_type_adapter.h index f2fc43a8a..eb89a467c 100644 --- a/eval/public/structs/proto_message_type_adapter.h +++ b/eval/public/structs/proto_message_type_adapter.h @@ -26,6 +26,7 @@ #include "eval/public/cel_value.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" namespace google::api::expr::runtime { @@ -119,6 +120,13 @@ class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, const google::protobuf::Descriptor* descriptor_; }; +// Creates a CelValue from the given field on the proto message. This is the +// shared implementation for ProtoMessageTypeAdapter and +// DucktypedMessageAdapter. +absl::StatusOr CreateCelValueFromField( + const google::protobuf::Message* message, const google::protobuf::FieldDescriptor* field_desc, + ProtoWrapperTypeOptions unboxing_option, google::protobuf::Arena* arena); + // Returns a TypeInfo provider representing an arbitrary message. // This allows for the legacy duck-typed behavior of messages on field access // instead of expecting a particular message type given a TypeInfo. diff --git a/eval/public/structs/proto_message_type_adapter_test.cc b/eval/public/structs/proto_message_type_adapter_test.cc index e28d76102..c0e60c632 100644 --- a/eval/public/structs/proto_message_type_adapter_test.cc +++ b/eval/public/structs/proto_message_type_adapter_test.cc @@ -20,7 +20,9 @@ #include "google/protobuf/descriptor.pb.h" #include "absl/status/status.h" #include "base/attribute.h" +#include "common/legacy_value.h" #include "common/value.h" +#include "common/value_testing.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" @@ -41,6 +43,7 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::cel::ProtoWrapperTypeOptions; @@ -1407,5 +1410,57 @@ TEST(ProtoMesssageTypeAdapter, QualifyMapIndexLeafWrongType) { HasSubstr("Invalid map key type")))))); } +TEST(ProtoMesssageTypeAdapter, InteropUnwrappingNotGeneric) { + google::protobuf::Arena arena; + ProtoMessageTypeAdapter adapter( + google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( + "google.api.expr.runtime.TestMessage"), + google::protobuf::MessageFactory::generated_factory()); + + TestMessage message; + message.set_string_value("hello"); + auto legacy_value = CelValue::CreateMessageWrapper( + CelValue::MessageWrapper(&message, &adapter)); + cel::Value modern_value; + ASSERT_THAT(cel::ModernValue(&arena, legacy_value, modern_value), IsOk()); + auto unwrapped = cel::interop_internal::GetLegacyMessage(modern_value); + + // Can't unwrap a non-generic MessageWrapper -- we test by identity to + // be sure we're not dropping a custom adapter. + ASSERT_EQ(unwrapped, nullptr); +} + +TEST(ProtoMesssageTypeAdapter, InteropUnwrappingGeneric) { + google::protobuf::Arena arena; + + TestMessage message; + message.set_string_value("hello"); + auto legacy_value = CelValue::CreateMessageWrapper( + CelValue::MessageWrapper(&message, &GetGenericProtoTypeInfoInstance())); + cel::Value modern_value; + ASSERT_THAT(cel::ModernValue(&arena, legacy_value, modern_value), IsOk()); + auto unwrapped = cel::interop_internal::GetLegacyMessage(modern_value); + + ASSERT_EQ(unwrapped, &message); +} + +TEST(ProtoMesssageTypeAdapter, InteropFieldAccess) { + google::protobuf::Arena arena; + + TestMessage message; + message.set_string_value("hello"); + + const google::protobuf::FieldDescriptor* field = + message.GetDescriptor()->FindFieldByName("string_value"); + ASSERT_NE(field, nullptr); + cel::Value field_value; + ASSERT_THAT(cel::interop_internal::WrapLegacyMessageField( + &message, field, ProtoWrapperTypeOptions::kUnsetNull, &arena, + &field_value), + IsOk()); + + EXPECT_THAT(field_value, cel::test::StringValueIs("hello")); +} + } // namespace } // namespace google::api::expr::runtime diff --git a/eval/public/structs/protobuf_descriptor_type_provider_test.cc b/eval/public/structs/protobuf_descriptor_type_provider_test.cc index 3a8fae26b..aabdc38c4 100644 --- a/eval/public/structs/protobuf_descriptor_type_provider_test.cc +++ b/eval/public/structs/protobuf_descriptor_type_provider_test.cc @@ -17,16 +17,23 @@ #include #include "google/protobuf/wrappers.pb.h" +#include "absl/status/status_matchers.h" +#include "common/type.h" #include "eval/public/cel_value.h" #include "eval/public/structs/legacy_type_info_apis.h" #include "eval/public/testing/matchers.h" #include "extensions/protobuf/memory_manager.h" #include "internal/testing.h" +#include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; +using ::cel::expr::conformance::proto3::TestAllTypes; using ::cel::extensions::ProtoMemoryManager; TEST(ProtobufDescriptorProvider, Basic) { @@ -36,7 +43,7 @@ TEST(ProtobufDescriptorProvider, Basic) { google::protobuf::Arena arena; auto manager = ProtoMemoryManager(&arena); auto type_adapter = provider.ProvideLegacyType("google.protobuf.Int64Value"); - absl::optional type_info = + std::optional type_info = provider.ProvideLegacyTypeInfo("google.protobuf.Int64Value"); ASSERT_TRUE(type_adapter.has_value()); @@ -53,8 +60,9 @@ TEST(ProtobufDescriptorProvider, Basic) { ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder value, type_adapter->mutation_apis()->NewInstance(manager)); - ASSERT_OK(type_adapter->mutation_apis()->SetField( - "value", CelValue::CreateInt64(10), manager, value)); + ASSERT_THAT(type_adapter->mutation_apis()->SetField( + "value", CelValue::CreateInt64(10), manager, value), + IsOk()); ASSERT_OK_AND_ASSIGN( CelValue adapted, @@ -91,5 +99,60 @@ TEST(ProtobufDescriptorProvider, NotFound) { ASSERT_FALSE(type_info.has_value()); } +TEST(ProtobufDescriptorProvider, FindType) { + ProtobufDescriptorProvider provider( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory()); + ASSERT_OK_AND_ASSIGN(std::optional wrapper_type, + provider.FindType("google.protobuf.Int64Value")); + ASSERT_TRUE(wrapper_type.has_value()); + EXPECT_TRUE(wrapper_type->Is()); + EXPECT_EQ(wrapper_type->name(), "google.protobuf.Int64Value"); + + ASSERT_OK_AND_ASSIGN( + std::optional msg_type, + provider.FindType("cel.expr.conformance.proto3.TestAllTypes")); + ASSERT_TRUE(msg_type.has_value()); + EXPECT_TRUE(msg_type->Is()); + EXPECT_EQ(msg_type->name(), "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(ProtobufDescriptorProvider, FindStructTypeFieldByName) { + ProtobufDescriptorProvider provider( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory()); + ASSERT_OK_AND_ASSIGN(std::optional field, + provider.FindStructTypeFieldByName( + "google.protobuf.Int64Value", "value")); + ASSERT_TRUE(field.has_value()); + EXPECT_EQ(field->name(), "value"); + EXPECT_EQ(field->number(), 1); + EXPECT_EQ(field->GetType(), cel::IntType()); +} + +TEST(ProtobufDescriptorProvider, FindTypeNotFound) { + ProtobufDescriptorProvider provider( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory()); + ASSERT_OK_AND_ASSIGN(std::optional type, + provider.FindType("UnknownType")); + EXPECT_FALSE(type.has_value()); +} + +TEST(ProtobufDescriptorProvider, FindStructTypeFieldByNameNotFound) { + ProtobufDescriptorProvider provider( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory()); + ASSERT_OK_AND_ASSIGN(std::optional field, + provider.FindStructTypeFieldByName( + "google.protobuf.Int64Value", "unknown_field")); + EXPECT_FALSE(field.has_value()); + + ASSERT_OK_AND_ASSIGN( + std::optional field2, + provider.FindStructTypeFieldByName("UnknownType", "value")); + EXPECT_FALSE(field2.has_value()); +} + } // namespace } // namespace google::api::expr::runtime diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index 1223ff6d1..f1141ef08 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -174,6 +174,7 @@ cc_library( "//internal:testing_message_factory", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:die_if_null", "@com_google_protobuf//:protobuf", ], ) diff --git a/runtime/internal/runtime_env_testing.h b/runtime/internal/runtime_env_testing.h index 71b2096cd..8c391784d 100644 --- a/runtime/internal/runtime_env_testing.h +++ b/runtime/internal/runtime_env_testing.h @@ -18,12 +18,43 @@ #include #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/log/die_if_null.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" #include "runtime/internal/runtime_env.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" namespace cel::runtime_internal { absl_nonnull std::shared_ptr NewTestingRuntimeEnv(); +template +const google::protobuf::Descriptor* absl_nonnull GetTestingEnvDescriptor() { + const google::protobuf::Descriptor* descriptor = + internal::GetTestingDescriptorPool()->FindMessageTypeByName( + T::descriptor()->full_name()); + ABSL_CHECK(descriptor != nullptr) + << "Could not find CEL test env descriptor for type " + << T::descriptor()->full_name(); + return descriptor; +} + +template +google::protobuf::Message* absl_nonnull MakeTestingEnvDynamicProto( + const T& in, google::protobuf::Arena* absl_nonnull arena) { + const google::protobuf::Descriptor* descriptor = GetTestingEnvDescriptor(); + google::protobuf::Message* out = + ABSL_DIE_IF_NULL( + internal::GetTestingMessageFactory()->GetPrototype(descriptor)) + ->New(arena); + + ABSL_CHECK(out->MergeFromString(in.SerializeAsString())); + return out; +} + } // namespace cel::runtime_internal #endif // THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_RUNTIME_ENV_TESTING_H_ From cc92e62b71a8d9ef6c7eb73b9b409c9f9df15b14 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 23 Jul 2026 15:52:23 -0700 Subject: [PATCH 130/350] [Pratt Parser] Add Lexer PiperOrigin-RevId: 953000456 --- parser/internal/BUILD | 27 ++ parser/internal/lexer.cc | 711 ++++++++++++++++++++++++++++++++++ parser/internal/lexer.h | 286 ++++++++++++++ parser/internal/lexer_test.cc | 499 ++++++++++++++++++++++++ parser/internal/options.h | 4 +- parser/options.h | 20 +- 6 files changed, 1535 insertions(+), 12 deletions(-) create mode 100644 parser/internal/lexer.cc create mode 100644 parser/internal/lexer.h create mode 100644 parser/internal/lexer_test.cc diff --git a/parser/internal/BUILD b/parser/internal/BUILD index af815588e..c5bb23a4e 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -13,6 +13,7 @@ # limitations under the License. load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") load("//bazel:antlr.bzl", "antlr_cc_library") package(default_visibility = ["//visibility:public"]) @@ -29,3 +30,29 @@ antlr_cc_library( src = "Cel.g4", package = "cel_parser_internal", ) + +cc_library( + name = "lexer", + srcs = ["lexer.cc"], + hdrs = ["lexer.h"], + deps = [ + "//common:source", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) + +cc_test( + name = "lexer_test", + srcs = ["lexer_test.cc"], + deps = [ + ":lexer", + "//common:source", + "//internal:testing", + ], +) diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc new file mode 100644 index 000000000..db08b644f --- /dev/null +++ b/parser/internal/lexer.cc @@ -0,0 +1,711 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" +#include "absl/container/flat_hash_map.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace cel::parser_internal { + +namespace { + +[[nodiscard]] bool IsIdentTrailing(char32_t c) { + return c <= 0x7f && (absl::ascii_isdigit(static_cast(c)) || + absl::ascii_isalpha(static_cast(c)) || c == '_'); +} + +[[nodiscard]] bool IsPlusOrMinus(char32_t c) { return c == '+' || c == '-'; } + +[[nodiscard]] const absl::flat_hash_map& +Keywords() { + static const absl::NoDestructor< + absl::flat_hash_map> + kKeywords({ + {"false", TokenType::kFalse}, + {"true", TokenType::kTrue}, + {"null", TokenType::kNull}, + {"in", TokenType::kIn}, + {"as", TokenType::kReservedWord}, + {"break", TokenType::kReservedWord}, + {"const", TokenType::kReservedWord}, + {"continue", TokenType::kReservedWord}, + {"else", TokenType::kReservedWord}, + {"for", TokenType::kReservedWord}, + {"function", TokenType::kReservedWord}, + {"if", TokenType::kReservedWord}, + {"import", TokenType::kReservedWord}, + {"let", TokenType::kReservedWord}, + {"loop", TokenType::kReservedWord}, + {"package", TokenType::kReservedWord}, + {"namespace", TokenType::kReservedWord}, + {"return", TokenType::kReservedWord}, + {"var", TokenType::kReservedWord}, + {"void", TokenType::kReservedWord}, + {"while", TokenType::kReservedWord}, + }); + return *kKeywords; +} + +} // namespace + +std::string_view TokenTypeToString(TokenType type) { + switch (type) { + case TokenType::kError: + return "error"; + case TokenType::kEnd: + return "end"; + case TokenType::kWhitespace: + return "whitespace"; + case TokenType::kComment: + return "comment"; + case TokenType::kNull: + return "null"; + case TokenType::kFalse: + return "false"; + case TokenType::kTrue: + return "true"; + case TokenType::kIn: + return "in"; + case TokenType::kReservedWord: + return "reserved_word"; + case TokenType::kInt: + return "int"; + case TokenType::kUint: + return "uint"; + case TokenType::kFloat: + return "float"; + case TokenType::kString: + return "string"; + case TokenType::kBytes: + return "bytes"; + case TokenType::kIdent: + return "ident"; + case TokenType::kLeftBracket: + return "["; + case TokenType::kRightBracket: + return "]"; + case TokenType::kLeftBrace: + return "{"; + case TokenType::kRightBrace: + return "}"; + case TokenType::kLeftParen: + return "("; + case TokenType::kRightParen: + return ")"; + case TokenType::kDot: + return "."; + case TokenType::kComma: + return ","; + case TokenType::kMinus: + return "-"; + case TokenType::kPlus: + return "+"; + case TokenType::kAsterisk: + return "*"; + case TokenType::kSlash: + return "/"; + case TokenType::kPercent: + return "%"; + case TokenType::kQuestion: + return "?"; + case TokenType::kColon: + return ":"; + case TokenType::kExclamation: + return "!"; + case TokenType::kEqual: + return "="; + case TokenType::kEqualEqual: + return "=="; + case TokenType::kExclamationEqual: + return "!="; + case TokenType::kLess: + return "<"; + case TokenType::kLessEqual: + return "<="; + case TokenType::kGreater: + return ">"; + case TokenType::kGreaterEqual: + return ">="; + case TokenType::kLogicalAnd: + return "&&"; + case TokenType::kLogicalOr: + return "||"; + default: + return ""; + } +} + +Token Lexer::Lex() { + int32_t start = GetPosition(); + if (ABSL_PREDICT_FALSE(position_ >= content_.size())) { + at_end_ = true; + done_ = true; + return MakeToken(TokenType::kEnd, start, start); + } + char32_t c = content_.at(position_); + switch (c) { + case '\f': + ABSL_FALLTHROUGH_INTENDED; + case '\v': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\n': + ABSL_FALLTHROUGH_INTENDED; + case ' ': { + ConsumeWhitespace(); + return MakeToken(TokenType::kWhitespace, start, GetPosition()); + } + case '.': { + if (position_ + 1 < content_.size() && + content_.at(position_ + 1) <= 0x7f && + absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { + return ConsumeNumericLiteral(); + } + Advance(1); + return MakeToken(TokenType::kDot, start, GetPosition()); + } + case ',': { + Advance(1); + return MakeToken(TokenType::kComma, start, GetPosition()); + } + case '!': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kExclamationEqual, start, GetPosition()); + } + return MakeToken(TokenType::kExclamation, start, GetPosition()); + } + case '?': { + Advance(1); + return MakeToken(TokenType::kQuestion, start, GetPosition()); + } + case '(': { + Advance(1); + return MakeToken(TokenType::kLeftParen, start, GetPosition()); + } + case ')': { + Advance(1); + return MakeToken(TokenType::kRightParen, start, GetPosition()); + } + case '{': { + Advance(1); + return MakeToken(TokenType::kLeftBrace, start, GetPosition()); + } + case '}': { + Advance(1); + return MakeToken(TokenType::kRightBrace, start, GetPosition()); + } + case '[': { + Advance(1); + return MakeToken(TokenType::kLeftBracket, start, GetPosition()); + } + case ']': { + Advance(1); + return MakeToken(TokenType::kRightBracket, start, GetPosition()); + } + case '=': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kEqualEqual, start, GetPosition()); + } + return MakeToken(TokenType::kEqual, start, GetPosition()); + } + case '<': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kLessEqual, start, GetPosition()); + } + return MakeToken(TokenType::kLess, start, GetPosition()); + } + case '>': { + Advance(1); + if (Consume('=')) { + return MakeToken(TokenType::kGreaterEqual, start, GetPosition()); + } + return MakeToken(TokenType::kGreater, start, GetPosition()); + } + case ':': { + Advance(1); + return MakeToken(TokenType::kColon, start, GetPosition()); + } + case '%': { + Advance(1); + return MakeToken(TokenType::kPercent, start, GetPosition()); + } + case '+': { + Advance(1); + return MakeToken(TokenType::kPlus, start, GetPosition()); + } + case '-': { + Advance(1); + return MakeToken(TokenType::kMinus, start, GetPosition()); + } + case '*': { + Advance(1); + return MakeToken(TokenType::kAsterisk, start, GetPosition()); + } + case '/': { + Advance(1); + if (Consume('/')) { + ConsumeLine(); + return MakeToken(TokenType::kComment, start, GetPosition()); + } + return MakeToken(TokenType::kSlash, start, GetPosition()); + } + case '&': { + Advance(1); + if (Consume('&')) { + return MakeToken(TokenType::kLogicalAnd, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '&', expected '&&'"); + } + case '|': { + Advance(1); + if (Consume('|')) { + return MakeToken(TokenType::kLogicalOr, start, GetPosition()); + } + return SetError(start, GetPosition(), + "unexpected single '|', expected '||'"); + } + case '_': { + return ConsumeIdent(); + } + case '`': { + return ConsumeQuotedIdent(); + } + case '\'': { + return ConsumeStringLiteral(start, '\''); + } + case '"': { + return ConsumeStringLiteral(start, '"'); + } + case 'r': + ABSL_FALLTHROUGH_INTENDED; + case 'R': + ABSL_FALLTHROUGH_INTENDED; + case 'b': + ABSL_FALLTHROUGH_INTENDED; + case 'B': { + if (auto token = ConsumePrefixedStringLiteral(); token.has_value()) { + return *token; + } + break; + } + default: + break; + } + if (c <= 0x7f && absl::ascii_isdigit(static_cast(c))) { + return ConsumeNumericLiteral(); + } + if (c <= 0x7f && absl::ascii_isalpha(static_cast(c))) { + // Root identifiers (the ones starting with a period) are returned as + // a sequence of kDot and kIdent tokens. + return ConsumeIdent(); + } + Advance(1); + return SetError(start, GetPosition(), "unexpected character"); +} + +// Consumes characters up to and including the first occurrence of character `c` +// without interpreting backslashes as escapes. +// Returns true if `c` was found and consumed; false if end of input was +// reached. +bool Lexer::ConsumeUntilAfter(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + for (int32_t pos = position_; pos < content_.size(); ++pos) { + if (content_.at(pos) == c) { + AdvanceProcessingNewLines(pos + 1); + return true; + } + } + AdvanceProcessingNewLines(content_.size()); + return false; +} + +// Consumes characters up to and including the first occurrence of substring `s` +// without interpreting backslashes as escapes (`s` must not contain newlines). +// Returns true if `s` was found and consumed; false if end of input was +// reached. +bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + int32_t pos = position_; + while (pos + static_cast(s.size()) <= content_.size()) { + bool match = true; + for (size_t i = 0; i < s.size(); ++i) { + if (content_.at(pos + static_cast(i)) != s[i]) { + match = false; + break; + } + } + if (match) { + AdvanceProcessingNewLines(pos + static_cast(s.size())); + return true; + } + ++pos; + } + AdvanceProcessingNewLines(content_.size()); + return false; +} + +// Consumes characters up to and including the first occurrence of `c` that is +// not preceded by an odd number of backslash ('\') escape characters. Returns +// true if an unescaped `c` was found and consumed; false if reached EOF. +bool Lexer::ConsumeUntilAfterUnescaped(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + ABSL_DCHECK_NE(c, '\\'); + int32_t pos = position_; + bool escaped = false; + while (pos < content_.size()) { + char32_t cc = content_.at(pos); + if (cc == '\\') { + escaped = !escaped; + } else { + if (cc == c && !escaped) { + AdvanceProcessingNewLines(pos + 1); + return true; + } + escaped = false; + } + ++pos; + } + AdvanceProcessingNewLines(content_.size()); + return false; +} + +// Consumes characters up to and including the first occurrence of substring `s` +// where the first character of `s` is not preceded by an odd number of +// backslashes. Returns true if an unescaped `s` was found and consumed; false +// if reached EOF. +bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + int32_t pos = position_; + bool escaped = false; + while (pos < content_.size()) { + char32_t cc = content_.at(pos); + if (cc == '\\') { + escaped = !escaped; + } else { + if (!escaped && pos + static_cast(s.size()) <= content_.size()) { + bool match = true; + for (size_t j = 0; j < s.size(); ++j) { + if (content_.at(pos + static_cast(j)) != s[j]) { + match = false; + break; + } + } + if (match) { + AdvanceProcessingNewLines(pos + static_cast(s.size())); + return true; + } + } + escaped = false; + } + ++pos; + } + AdvanceProcessingNewLines(content_.size()); + return false; +} + +bool Lexer::MatchString(std::u32string_view s) const { + if (position_ + static_cast(s.size()) > content_.size()) { + return false; + } + for (size_t i = 0; i < s.size(); ++i) { + if (content_.at(position_ + static_cast(i)) != s[i]) { + return false; + } + } + return true; +} + +std::optional Lexer::MatchIf( + absl::FunctionRef predicate) const { + if (position_ < content_.size()) { + char32_t cp = content_.at(position_); + if (predicate(cp)) { + return cp; + } + } + return std::nullopt; +} + +void Lexer::ConsumeLine() { + while (position_ < content_.size()) { + if (content_.at(position_) == '\n') { + Advance(1); + return; + } + Advance(1); + } +} + +void Lexer::ConsumeWhitespace() { + while (position_ < content_.size()) { + char32_t c = content_.at(position_); + switch (c) { + case '\f': + ABSL_FALLTHROUGH_INTENDED; + case '\n': + ABSL_FALLTHROUGH_INTENDED; + case ' ': + ABSL_FALLTHROUGH_INTENDED; + case '\r': + ABSL_FALLTHROUGH_INTENDED; + case '\v': + ABSL_FALLTHROUGH_INTENDED; + case '\t': + Advance(1); + break; + default: + return; + } + } +} + +bool Lexer::Consume(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + if (Match(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeIgnoreCase(char32_t c) { + ABSL_DCHECK_NE(c, '\n'); + if (MatchIgnoreCase(c)) { + Advance(1); + return true; + } + return false; +} + +bool Lexer::ConsumeString(std::u32string_view s) { + ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); + if (MatchString(s)) { + Advance(s.size()); + return true; + } + return false; +} + +std::optional Lexer::ConsumeIf( + absl::FunctionRef predicate) { + std::optional match = MatchIf(predicate); + if (match.has_value()) { + ABSL_DCHECK_NE(*match, '\n'); + Advance(1); + } + return match; +} + +bool Lexer::ConsumeDigits() { + bool advanced = false; + while (position_ < content_.size()) { + char32_t c = content_.at(position_); + if (c > 0x7f || !absl::ascii_isdigit(static_cast(c))) { + break; + } + Advance(1); + advanced = true; + } + return advanced; +} + +bool Lexer::ConsumeHexDigits() { + bool advanced = false; + while (position_ < content_.size()) { + char32_t c = content_.at(position_); + if (c > 0x7f || !absl::ascii_isxdigit(static_cast(c))) { + break; + } + Advance(1); + advanced = true; + } + return advanced; +} + +TokenType Lexer::ConsumeIntegralSuffix() { + if (ConsumeIgnoreCase('u')) { + return TokenType::kUint; + } + return TokenType::kInt; +} + +Token Lexer::ConsumeQuotedIdent() { + int32_t start = GetPosition(); + Advance(1); + if (!ConsumeUntilAfter('`')) { + return SetError(start, GetPosition(), "unterminated quoted identifier"); + } + return MakeToken(TokenType::kIdent, start, GetPosition()); +} + +Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes, + bool is_raw) { + Advance(1); + std::u32string triple_quote(3, quote); + if (ConsumeString(std::u32string_view(triple_quote.data(), 2))) { + if (is_raw ? !ConsumeUntilAfterString(triple_quote) + : !ConsumeUntilAfterUnescapedString(triple_quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, start, + GetPosition()); + } + if (is_raw ? !ConsumeUntilAfter(quote) : !ConsumeUntilAfterUnescaped(quote)) { + return SetError(start, GetPosition(), + is_bytes ? "unterminated bytes literal" + : "unterminated string literal"); + } + return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, start, + GetPosition()); +} + +// Consumes prefixed string and bytes literals. +// Handles the following prefix sequences (case-insensitive for 'r' and 'b'): +// - Raw strings: r"...", r'...', r"""...""", r'''...''' +// - Bytes: b"...", b'...', b"""...""", b'''...''' +// - Raw bytes: br"...", br'...', br"""...""", br'''...''', rb"...", rb'...', +// rb"""...""", rb'''...''' +std::optional Lexer::ConsumePrefixedStringLiteral() { + int32_t start = GetPosition(); + if (position_ >= content_.size()) return std::nullopt; + char32_t c = content_.at(position_); + bool is_bytes = (c == 'b' || c == 'B'); + bool is_raw = (c == 'r' || c == 'R'); + size_t lookahead = 1; + if (position_ + 1 < content_.size()) { + char32_t c2 = content_.at(position_ + 1); + if ((is_bytes && (c2 == 'r' || c2 == 'R')) || + (!is_bytes && (c2 == 'b' || c2 == 'B'))) { + is_bytes = true; + is_raw = true; + lookahead = 2; + } + } + if (position_ + static_cast(lookahead) < content_.size()) { + char32_t quote = content_.at(position_ + static_cast(lookahead)); + if (quote == '"' || quote == '\'') { + Advance(lookahead); + return ConsumeStringLiteral(start, quote, is_bytes, is_raw); + } + } + return std::nullopt; +} + +// Consumes a numeric literal token and returns its TokenType (kInt, kUint, or +// kFloat). Recognizes the following literal formats: +// - Hexadecimal integers (kInt / kUint): 0x1A, 0XFFu, 0x0U +// - Decimal integers (kInt / kUint): 0, 45U, 123456 +// - Floating-point numbers (kFloat): .12345, 1.23, 1e6, 1.5e+10, .5e-3 +Token Lexer::ConsumeNumericLiteral() { + int32_t start = GetPosition(); + char32_t c = content_.at(position_); + bool floating_point = false; + if (c == '.') { + floating_point = true; + Advance(1); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after decimal separator"); + } + } else { + Advance(1); + if (c == '0') { + if (ConsumeIgnoreCase('x')) { + if (!ConsumeHexDigits()) { + return SetError( + start, GetPosition(), + "integral literal missing digits after hexadecimal separator"); + } + auto token_type = ConsumeIntegralSuffix(); + if (ConsumeIf(IsIdentTrailing)) { + return SetError( + start, GetPosition(), + absl::StrCat(TokenTypeToString(token_type), + " literal has unexpected trailing characters")); + } + return MakeToken(token_type, start, GetPosition()); + } + } + static_cast(ConsumeDigits()); + if (position_ < content_.size() && content_.at(position_) == '.' && + position_ + 1 < content_.size() && content_.at(position_ + 1) <= 0x7f && + absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { + floating_point = true; + Advance(1); + static_cast(ConsumeDigits()); + } + } + if (ConsumeIgnoreCase('e')) { + floating_point = true; + static_cast(ConsumeIf(IsPlusOrMinus)); + if (!ConsumeDigits()) { + return SetError( + start, GetPosition(), + "floating point literal missing digits after exponent separator"); + } + } + auto token_type = + floating_point ? TokenType::kFloat : ConsumeIntegralSuffix(); + if (ConsumeIf(IsIdentTrailing)) { + return SetError( + start, GetPosition(), + absl::StrCat(TokenTypeToString(token_type), + " literal has unexpected trailing characters")); + } + return MakeToken(token_type, start, GetPosition()); +} + +Token Lexer::ConsumeIdent() { + int32_t start = GetPosition(); + while (position_ < content_.size()) { + char32_t c = content_.at(position_); + if (!IsIdentTrailing(c)) { + break; + } + Advance(1); + } + int32_t end = GetPosition(); + std::string word = content_.ToString(start, end); + const auto& keywords = Keywords(); + if (auto it = keywords.find(word); it != keywords.end()) { + return MakeToken(it->second, start, end); + } + return MakeToken(TokenType::kIdent, start, end); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h new file mode 100644 index 000000000..ef166d4bb --- /dev/null +++ b/parser/internal/lexer.h @@ -0,0 +1,286 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/base/optimization.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/strings/ascii.h" +#include "common/source.h" + +namespace cel::parser_internal { + +enum class TokenType { + kError = 0, + kEnd, + kWhitespace, + kComment, + + // Keywords + kNull, + kFalse, + kTrue, + kIn, + kReservedWord, + + // Literals + kInt, + kUint, + kFloat, + kString, + kBytes, + + // Identifiers (standard bare identifiers and backtick-quoted identifiers). + // Note: The lexer does not validate whether a quoted/escaped identifier is + // source-legal or permitted in its syntactic context. Because + // non-source-legal identifiers are used internally in macros and functions, + // the parser must strictly validate the characters inside quoted identifiers + // and verify that they only appear where permitted (e.g., field selections + // and struct field specifiers). + kIdent, + + // Delimiters + kLeftBracket, // [ + kRightBracket, // ] + kLeftBrace, // { + kRightBrace, // } + kLeftParen, // ( + kRightParen, // ) + + // Operators + kDot, // . + kComma, // , + kMinus, // - + kPlus, // + + kAsterisk, // * + kSlash, // / + kPercent, // % + kQuestion, // ? + kColon, // : + kExclamation, // ! + kEqual, // = + kEqualEqual, // == + kExclamationEqual, // != + kLess, // < + kLessEqual, // <= + kGreater, // > + kGreaterEqual, // >= + kLogicalAnd, // && + kLogicalOr, // || +}; + +ABSL_ATTRIBUTE_PURE_FUNCTION std::string_view TokenTypeToString(TokenType type); + +struct Token final { + TokenType type = TokenType::kError; + int32_t start = 0; + int32_t end = 0; +}; + +struct LexerError final { + int32_t start = 0; + int32_t end = 0; + std::string message; +}; + +// Lexer performs fast tokenization of CEL expression source code. +// +// Responsibilities & Parser Expectations: +// This lexer is designed for speed and does not perform comprehensive semantic +// or syntax validation: +// +// 1. String and Bytes Literal Escape Sequences: +// - For standard single- and double-quoted literals ("..." and '...'), the +// lexer recognizes backslash ('\') only to determine whether the closing +// delimiter ('"' or '\'') is escaped (e.g., \" and \' do not terminate the +// literal, whereas \\" terminates it because the backslash is escaped). +// - For triple-quoted literals ("""...""" and '''...''') and raw literals +// (r"...", r'''...'''), backslashes and escape sequences are not processed +// when locating the closing delimiter. +// - The lexer does NOT validate, decode, or check the syntax of any escape +// sequences (e.g., \n, \r, \t, \xHH, \uHHHH, \U00HHHHHH, \0, or invalid +// escapes like \q). All characters and backslashes within the literal +// boundaries are preserved verbatim in the token's text span. +// - The parser/caller is strictly responsible for validating and unescaping +// all escape sequences and reporting syntax errors for invalid escape +// sequences when converting string and bytes tokens during AST +// construction. +// +// 2. Numeric Literals: +// - Performs only general bounds and format matching for integers and +// floating-point numeric literals. The lexer expects the parser to perform +// final validation and numeric conversion when building the AST. +class Lexer final { + public: + explicit Lexer(const cel::Source& source) + : content_(source.content()), position_(0) { + ABSL_DCHECK_LE(content_.size(), static_cast( + std::numeric_limits::max())); + } + + Lexer(const Lexer&) = delete; + Lexer(Lexer&&) = delete; + Lexer& operator=(const Lexer&) = delete; + Lexer& operator=(Lexer&&) = delete; + + // Scans and returns the next token from the source. + [[nodiscard]] ABSL_ATTRIBUTE_NOINLINE Token Lex(); + + // Inspect the error from the last call to `Lex()` that returned an error + // token. The reference is not guaranteed to be valid after further calls to + // `Lex`. + [[nodiscard]] const LexerError& GetError() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return error_; + } + + [[nodiscard]] int32_t GetPosition() const { return position_; } + + private: + [[nodiscard]] bool Match(char32_t c) const { + return position_ < content_.size() && content_.at(position_) == c; + } + + [[nodiscard]] bool MatchIgnoreCase(char32_t c) const { + if (position_ >= content_.size()) return false; + char32_t cp = content_.at(position_); + return cp <= 0x7f && c <= 0x7f && + absl::ascii_tolower(static_cast(cp)) == + absl::ascii_tolower(static_cast(c)); + } + + void Advance(size_t n) { + ABSL_DCHECK_LE(n, static_cast(content_.size() - position_)); + position_ += static_cast(n); + } + + void AdvanceProcessingNewLines(int32_t end_position) { + ABSL_DCHECK_LE(end_position, content_.size()); + ABSL_DCHECK_GE(end_position, position_); + Advance(static_cast(end_position - position_)); + } + + [[nodiscard]] Token MakeToken(TokenType type, int32_t start, int32_t end) { + if (ABSL_PREDICT_FALSE(at_end_)) { + AtEndTokenCreated(); + } + return Token{.type = type, .start = start, .end = end}; + } + + [[nodiscard]] Token SetError(int32_t start, int32_t end, + std::string message) { + error_ = + LexerError{.start = start, .end = end, .message = std::move(message)}; + return Token{.type = TokenType::kError, .start = start, .end = end}; + } + + void AtEndTokenCreated() { done_ = true; } + + // Consumes characters up to and including the first occurrence of character + // `c` without interpreting backslashes as escapes. Returns true if `c` was + // found and consumed; false if end of input was reached. + [[nodiscard]] bool ConsumeUntilAfter(char32_t c); + + // Consumes characters up to and including the first occurrence of substring + // `s` without interpreting backslashes as escapes (`s` must not contain + // newlines). Returns true if `s` was found and consumed; false if end of + // input was reached. + [[nodiscard]] bool ConsumeUntilAfterString(std::u32string_view s); + + // Consumes characters up to and including the first occurrence of `c` that is + // not preceded by an odd number of backslash ('\') escape characters. Returns + // true if an unescaped `c` was found and consumed; false if reached EOF. + [[nodiscard]] bool ConsumeUntilAfterUnescaped(char32_t c); + + // Consumes characters up to and including the first occurrence of substring + // `s` where the first character of `s` is not preceded by an odd number of + // backslashes. Returns true if an unescaped `s` was found and consumed; false + // if reached EOF. + [[nodiscard]] bool ConsumeUntilAfterUnescapedString(std::u32string_view s); + + [[nodiscard]] bool MatchString(std::u32string_view s) const; + + [[nodiscard]] std::optional MatchIf( + absl::FunctionRef predicate) const; + + void ConsumeLine(); + + void ConsumeWhitespace(); + + [[nodiscard]] bool Consume(char32_t c); + + [[nodiscard]] bool ConsumeIgnoreCase(char32_t c); + + [[nodiscard]] bool ConsumeString(std::u32string_view s); + + [[nodiscard]] std::optional ConsumeIf( + absl::FunctionRef predicate); + + [[nodiscard]] bool ConsumeDigits(); + + [[nodiscard]] bool ConsumeHexDigits(); + + [[nodiscard]] TokenType ConsumeIntegralSuffix(); + + // Consumes a backtick-quoted identifier (`...`) and returns + // TokenType::kIdent. The token text preserves the surrounding backticks so + // the parser can detect quoted identifiers and enforce restrictions on their + // characters and allowed syntactic locations (such as field selections and + // struct field specifiers). + [[nodiscard]] Token ConsumeQuotedIdent(); + + [[nodiscard]] Token ConsumeStringLiteral(int32_t start, char32_t quote, + bool is_bytes = false, + bool is_raw = false); + + // Consumes prefixed string and bytes literals. + // Handles the following prefix sequences (case-insensitive for 'r' and 'b'): + // - Raw strings: r"...", r'...', r"""...""", r'''...''' + // - Bytes: b"...", b'...', b"""...""", b'''...''' + // - Raw bytes: br"...", br'...', br"""...""", br'''...''', rb"...", rb'...', + // rb"""...""", rb'''...''' + [[nodiscard]] std::optional ConsumePrefixedStringLiteral(); + + // Consumes a numeric literal token and returns its TokenType (kInt, kUint, or + // kFloat). Recognizes the following literal formats: + // - Decimal integers (kInt / kUint): 0, 45U, 123456 + // - Hexadecimal integers (kInt / kUint): 0x1A, 0XFFu, 0x0U + // - Floating-point numbers (kFloat): .12345, 1.23, 1e6, 1.5e+10, .5e-3 + [[nodiscard]] Token ConsumeNumericLiteral(); + + // Consumes an identifier token and checks if it matches any reserved + // keywords. + [[nodiscard]] Token ConsumeIdent(); + + cel::SourceContentView content_; + int32_t position_ = 0; + bool at_end_ = false; + bool done_ = false; + LexerError error_; +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_LEXER_H_ diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc new file mode 100644 index 000000000..ca311baf8 --- /dev/null +++ b/parser/internal/lexer_test.cc @@ -0,0 +1,499 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/lexer.h" + +#include +#include +#include +#include + +#include "common/source.h" +#include "internal/testing.h" + +namespace cel::parser_internal { +namespace { + +MATCHER_P3(IsToken, source, expected_type, expected_text, "") { + if (arg.type != expected_type) { + *result_listener << "type is " << TokenTypeToString(arg.type) + << " (expected " << TokenTypeToString(expected_type) + << ")"; + return false; + } + std::string actual_text = source->content().ToString(arg.start, arg.end); + if (actual_text != expected_text) { + *result_listener << "text is '" << actual_text << "' (expected '" + << expected_text << "')"; + return false; + } + return true; +} + +struct LexerTestCase { + std::string_view name; + std::string_view source; + std::vector> expected_tokens; +}; + +using LexerTest = testing::TestWithParam; + +TEST_P(LexerTest, LexesSuccessTokens) { + const LexerTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.source)); + Lexer lexer(*source); + + for (const auto& [type, text] : test_case.expected_tokens) { + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), type, text)); + } + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kEnd, "")); +} + +INSTANTIATE_TEST_SUITE_P( + LexerTest, LexerTest, + testing::ValuesIn({ + {"NullSource", std::string_view(nullptr, 0), {}}, + {"Empty", "", {}}, + {"Whitespace", " \n ", {{TokenType::kWhitespace, " \n "}}}, + {"KeywordsAndIdents", + "null false true in as return foo_bar _foo_bar _ `quoted.ident`", + {{TokenType::kNull, "null"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFalse, "false"}, + {TokenType::kWhitespace, " "}, + {TokenType::kTrue, "true"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIn, "in"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "as"}, + {TokenType::kWhitespace, " "}, + {TokenType::kReservedWord, "return"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_foo_bar"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "_"}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`quoted.ident`"}}}, + {"Numbers", + "123 45u 0x1A 3.14 .5 1e6 2.5e-3 45U 0x1Au 0x1AU", + {{TokenType::kInt, "123"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x1A"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "3.14"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, ".5"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "1e6"}, + {TokenType::kWhitespace, " "}, + {TokenType::kFloat, "2.5e-3"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "45U"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1Au"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0x1AU"}}}, + {"IntEOF", "123456", {{TokenType::kInt, "123456"}}}, + {"HexIntEOF", "0x1A2B", {{TokenType::kInt, "0x1A2B"}}}, + {"FloatPositiveExponentEOF", "1e+6", {{TokenType::kFloat, "1e+6"}}}, + {"FloatEOF", ".12345", {{TokenType::kFloat, ".12345"}}}, + {"IntDotIdent", + "1.foo", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kIdent, "foo"}}}, + {"IntDotWhitespace", + "1. ", + {{TokenType::kInt, "1"}, + {TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}}}, + {"IntDotEOF", "1.", {{TokenType::kInt, "1"}, {TokenType::kDot, "."}}}, + {"DotAtEOFBeforeDigit", + std::string_view(".6", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"DotAtEOFBeforeIdent", + std::string_view(".a", /*length=*/1), + {{TokenType::kDot, "."}}}, + {"ZeroNumbers", + "0 0u 0x0", + {{TokenType::kInt, "0"}, + {TokenType::kWhitespace, " "}, + {TokenType::kUint, "0u"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "0x0"}}}, + {"StringsAndBytes", + R"("hello" 'world' """ "allowed!" ""also allowed"" \"""also allowed""\" """ r"raw" b"bytes" rb'\x00' '''multi +single''' R"raw_upper" B"bytes_upper" b'''multi +bytes''' br"raw_bytes" `a.b-c/d e` +"\a\b\f\n\r\t\v\"\'\\\?\` \x1A \u00A0 \U0001F600 \012")", + {{TokenType::kString, "\"hello\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'world'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, + "\"\"\" \"allowed!\" \"\"also allowed\"\" \\\"\"\"also " + "allowed\"\"\\\" \"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"raw\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "rb'\\x00'"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "'''multi\nsingle'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "R\"raw_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "B\"bytes_upper\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b'''multi\nbytes'''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "br\"raw_bytes\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kIdent, "`a.b-c/d e`"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kString, + "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\\` \\x1A \\u00A0 \\U0001F600 " + "\\012\""}}}, + {"EmptyStrings", + "\"\" '' \"\"\"\"\"\" '''''' r\"\" r'' r\"\"\"\"\"\" r'''''' b\"\" " + "b'' b\"\"\"\"\"\" b''''''", + {{TokenType::kString, "\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r''''''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''"}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"\"\"\"\"\""}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b''''''"}}}, + {"OperatorsAndDelimiters", + ". , + - * / % == != < <= > >= && || ! ? : [] { } ( )", + {{TokenType::kDot, "."}, + {TokenType::kWhitespace, " "}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kPlus, "+"}, + {TokenType::kWhitespace, " "}, + {TokenType::kMinus, "-"}, + {TokenType::kWhitespace, " "}, + {TokenType::kAsterisk, "*"}, + {TokenType::kWhitespace, " "}, + {TokenType::kSlash, "/"}, + {TokenType::kWhitespace, " "}, + {TokenType::kPercent, "%"}, + {TokenType::kWhitespace, " "}, + {TokenType::kEqualEqual, "=="}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamationEqual, "!="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLess, "<"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLessEqual, "<="}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreater, ">"}, + {TokenType::kWhitespace, " "}, + {TokenType::kGreaterEqual, ">="}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalAnd, "&&"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLogicalOr, "||"}, + {TokenType::kWhitespace, " "}, + {TokenType::kExclamation, "!"}, + {TokenType::kWhitespace, " "}, + {TokenType::kQuestion, "?"}, + {TokenType::kWhitespace, " "}, + {TokenType::kColon, ":"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, + {TokenType::kRightBracket, "]"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightBrace, "}"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftParen, "("}, + {TokenType::kWhitespace, " "}, + {TokenType::kRightParen, ")"}}}, + {"Comments", + "a\n// comment\nb", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kComment, "// comment\n"}, + {TokenType::kIdent, "b"}}}, + {"CommentWithoutTrailingNewlineEOF", + "// comment without trailing newline", + {{TokenType::kComment, "// comment without trailing newline"}}}, + {"CommentAfterTokenWithoutTrailingNewlineEOF", + "a // comment without trailing newline", + {{TokenType::kIdent, "a"}, + {TokenType::kWhitespace, " "}, + {TokenType::kComment, "// comment without trailing newline"}}}, + {"NestedDelimiters", + "(((1 + 2) * [3, {4: 5}])))", + {{TokenType::kLeftParen, "("}, {TokenType::kLeftParen, "("}, + {TokenType::kLeftParen, "("}, {TokenType::kInt, "1"}, + {TokenType::kWhitespace, " "}, {TokenType::kPlus, "+"}, + {TokenType::kWhitespace, " "}, {TokenType::kInt, "2"}, + {TokenType::kRightParen, ")"}, {TokenType::kWhitespace, " "}, + {TokenType::kAsterisk, "*"}, {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, {TokenType::kInt, "3"}, + {TokenType::kComma, ","}, {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, {TokenType::kInt, "4"}, + {TokenType::kColon, ":"}, {TokenType::kWhitespace, " "}, + {TokenType::kInt, "5"}, {TokenType::kRightBrace, "}"}, + {TokenType::kRightBracket, "]"}, {TokenType::kRightParen, ")"}, + {TokenType::kRightParen, ")"}, {TokenType::kRightParen, ")"}}}, + {"NestedCommentsInDelimiters", + "(\n // leading comment\n [\n 1, // inline comment\n {2: 3}\n " + " ]\n)", + {{TokenType::kLeftParen, "("}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kComment, "// leading comment\n"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBracket, "["}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kInt, "1"}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kComment, "// inline comment\n"}, + {TokenType::kWhitespace, " "}, + {TokenType::kLeftBrace, "{"}, + {TokenType::kInt, "2"}, + {TokenType::kColon, ":"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "3"}, + {TokenType::kRightBrace, "}"}, + {TokenType::kWhitespace, "\n "}, + {TokenType::kRightBracket, "]"}, + {TokenType::kWhitespace, "\n"}, + {TokenType::kRightParen, ")"}}}, + {"ComplexNestedLiteralsAndDelimiters", + "`foo.bar`([\"nested\\\"quote\", r'''raw 'quotes' inside'''], " + "b\"bytes\")", + {{TokenType::kIdent, "`foo.bar`"}, + {TokenType::kLeftParen, "("}, + {TokenType::kLeftBracket, "["}, + {TokenType::kString, "\"nested\\\"quote\""}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kString, "r'''raw 'quotes' inside'''"}, + {TokenType::kRightBracket, "]"}, + {TokenType::kComma, ","}, + {TokenType::kWhitespace, " "}, + {TokenType::kBytes, "b\"bytes\""}, + {TokenType::kRightParen, ")"}}}, + }), + [](const testing::TestParamInfo& info) { + return std::string(info.param.name); + }); + +TEST(LexerTest, LineOffsets) { + std::string_view source_text = "a\n// comment\nb"; + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(source_text)); + Lexer lexer(*source); + + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kIdent, "a")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kComment, "// comment\n")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kIdent, "b")); + + auto line_offsets = source->line_offsets(); + ASSERT_GE(line_offsets.size(), 2); + EXPECT_EQ(line_offsets[0], 2); + EXPECT_EQ(line_offsets[1], 13); +} + +TEST(LexerTest, LineOffsetsInStringsAndIdentifiers) { + std::string_view source_text = + "'''multi\nline'''\n\"another\nline\"\n`ident\nhere`"; + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(source_text)); + Lexer lexer(*source); + + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kString, "'''multi\nline'''")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kString, "\"another\nline\"")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); + EXPECT_THAT(lexer.Lex(), + IsToken(source.get(), TokenType::kIdent, "`ident\nhere`")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kEnd, "")); + + auto line_offsets = source->line_offsets(); + ASSERT_GE(line_offsets.size(), 5); + EXPECT_EQ(line_offsets[0], 9); + EXPECT_EQ(line_offsets[1], 17); + EXPECT_EQ(line_offsets[2], 26); + EXPECT_EQ(line_offsets[3], 32); + EXPECT_EQ(line_offsets[4], 39); +} + +struct LexerErrorTestCase { + std::string_view source; + std::string_view expected_error_message; + std::string_view expected_error_location; +}; + +using LexerErrorTest = testing::TestWithParam; + +TEST_P(LexerErrorTest, LexesErrorTokenAndStoresError) { + const LexerErrorTestCase& test_case = GetParam(); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.source)); + Lexer lexer(*source); + Token token = lexer.Lex(); + while (token.type != TokenType::kError && token.type != TokenType::kEnd) { + token = lexer.Lex(); + } + EXPECT_EQ(token.type, TokenType::kError); + EXPECT_EQ(lexer.GetError().message, test_case.expected_error_message); + auto location = source->GetLocation(lexer.GetPosition()); + ASSERT_TRUE(location.has_value()); + EXPECT_EQ(source->DisplayErrorLocation(*location), + test_case.expected_error_location); +} + +INSTANTIATE_TEST_SUITE_P( + ErrorCases, LexerErrorTest, + testing::Values( + LexerErrorTestCase{ + .source = "\"unterminated", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | \"unterminated" + "\n | .............^", + }, + LexerErrorTestCase{ + .source = "0x", + .expected_error_message = + "integral literal missing digits after hexadecimal separator", + .expected_error_location = "\n | 0x" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "@", + .expected_error_message = "unexpected character", + .expected_error_location = "\n | @" + "\n | .^", + }, + LexerErrorTestCase{ + .source = "0x1A_invalid", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_error_location = "\n | 0x1A_invalid" + "\n | .....^", + }, + LexerErrorTestCase{ + .source = "123_invalid", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_error_location = "\n | 123_invalid" + "\n | ....^", + }, + LexerErrorTestCase{ + .source = "1x0", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_error_location = "\n | 1x0" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "2x", + .expected_error_message = + "int literal has unexpected trailing characters", + .expected_error_location = "\n | 2x" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "`unterminated quoted", + .expected_error_message = "unterminated quoted identifier", + .expected_error_location = "\n | `unterminated quoted" + "\n | ....................^", + }, + LexerErrorTestCase{ + .source = "'''unterminated multi", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | '''unterminated multi" + "\n | .....................^", + }, + LexerErrorTestCase{ + .source = "r'unterminated raw", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | r'unterminated raw" + "\n | ..................^", + }, + LexerErrorTestCase{ + .source = "b'unterminated bytes", + .expected_error_message = "unterminated bytes literal", + .expected_error_location = "\n | b'unterminated bytes" + "\n | ....................^", + }, + LexerErrorTestCase{ + .source = "1e", + .expected_error_message = + "floating point literal missing digits after exponent " + "separator", + .expected_error_location = "\n | 1e" + "\n | ..^", + }, + LexerErrorTestCase{ + .source = "\"😀😀😀😀😀\" ~error", + .expected_error_message = "unexpected character", + .expected_error_location = "\n | \"😀😀😀😀😀\" ~error" + "\n | .........^", + })); + +TEST(LexerErrorRecoveryTest, ResumesAfterError) { + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("1e, {2 3}")); + Lexer lexer(*source); + Token token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kError); + EXPECT_EQ(lexer.GetError().message, + "floating point literal missing digits after exponent separator"); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kComma); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kWhitespace); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kLeftBrace); + + token = lexer.Lex(); + EXPECT_EQ(token.type, TokenType::kInt); + EXPECT_EQ(token.start, 5); + EXPECT_EQ(token.end, 6); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/options.h b/parser/internal/options.h index ec2552204..4066d2619 100644 --- a/parser/internal/options.h +++ b/parser/internal/options.h @@ -15,7 +15,7 @@ #ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ -namespace cel_parser_internal { +namespace cel::parser_internal { inline constexpr int kDefaultErrorRecoveryLimit = 12; inline constexpr int kDefaultMaxRecursionDepth = 32; @@ -23,6 +23,6 @@ inline constexpr int kExpressionSizeCodepointLimit = 100'000; inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = 512; inline constexpr bool kDefaultAddMacroCalls = false; -} // namespace cel_parser_internal +} // namespace cel::parser_internal #endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_OPTIONS_H_ diff --git a/parser/options.h b/parser/options.h index 719bed454..22154e1f9 100644 --- a/parser/options.h +++ b/parser/options.h @@ -25,25 +25,25 @@ struct ParserOptions final { // Limit of the number of error recovery attempts made by the ANTLR parser // when processing an input. This limit, when reached, will halt further // parsing of the expression. - int error_recovery_limit = ::cel_parser_internal::kDefaultErrorRecoveryLimit; + int error_recovery_limit = ::cel::parser_internal::kDefaultErrorRecoveryLimit; // Limit on the amount of recursive parse instructions permitted when building // the abstract syntax tree for the expression. This prevents pathological // inputs from causing stack overflows. - int max_recursion_depth = ::cel_parser_internal::kDefaultMaxRecursionDepth; + int max_recursion_depth = ::cel::parser_internal::kDefaultMaxRecursionDepth; // Limit on the number of codepoints in the input string which the parser will // attempt to parse. int expression_size_codepoint_limit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; // Limit on the number of lookahead tokens to consume when attempting to // recover from an error. int error_recovery_token_lookahead_limit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; // Add macro calls to macro_calls list in source_info. - bool add_macro_calls = ::cel_parser_internal::kDefaultAddMacroCalls; + bool add_macro_calls = ::cel::parser_internal::kDefaultAddMacroCalls; // Enable support for optional syntax. bool enable_optional_syntax = false; @@ -76,20 +76,20 @@ using ParserOptions = ::cel::ParserOptions; ABSL_DEPRECATED("Use ParserOptions().error_recovery_limit instead.") inline constexpr int kDefaultErrorRecoveryLimit = - ::cel_parser_internal::kDefaultErrorRecoveryLimit; + ::cel::parser_internal::kDefaultErrorRecoveryLimit; ABSL_DEPRECATED("Use ParserOptions().max_recursion_depth instead.") inline constexpr int kDefaultMaxRecursionDepth = - ::cel_parser_internal::kDefaultMaxRecursionDepth; + ::cel::parser_internal::kDefaultMaxRecursionDepth; ABSL_DEPRECATED("Use ParserOptions().expression_size_codepoint_limit instead.") inline constexpr int kExpressionSizeCodepointLimit = - ::cel_parser_internal::kExpressionSizeCodepointLimit; + ::cel::parser_internal::kExpressionSizeCodepointLimit; ABSL_DEPRECATED( "Use ParserOptions().error_recovery_token_lookahead_limit instead.") inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = - ::cel_parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; ABSL_DEPRECATED("Use ParserOptions().add_macro_calls instead.") inline constexpr bool kDefaultAddMacroCalls = - ::cel_parser_internal::kDefaultAddMacroCalls; + ::cel::parser_internal::kDefaultAddMacroCalls; } // namespace google::api::expr::parser From 54a98f27cd23839737811d625ae4a465b31f1c9a Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 23 Jul 2026 16:29:01 -0700 Subject: [PATCH 131/350] [Pratt Parser] Add AstFactoryInterface template Also, add AstFactoryInterface specialization PiperOrigin-RevId: 953018238 --- common/expr_factory.h | 6 + parser/internal/BUILD | 28 ++++ parser/internal/ast_factory.h | 181 +++++++++++++++++++++ parser/internal/ast_factory_interface.h | 104 ++++++++++++ parser/internal/ast_factory_test.cc | 200 ++++++++++++++++++++++++ 5 files changed, 519 insertions(+) create mode 100644 parser/internal/ast_factory.h create mode 100644 parser/internal/ast_factory_interface.h create mode 100644 parser/internal/ast_factory_test.cc diff --git a/common/expr_factory.h b/common/expr_factory.h index 757318545..371361948 100644 --- a/common/expr_factory.h +++ b/common/expr_factory.h @@ -38,6 +38,11 @@ namespace tools { class ProtoToPredicateBuilder; } +namespace parser_internal { +template +class AstFactoryInterface; +} + class ExprFactory { protected: // `IsExprLike` determines whether `T` is some `Expr`. Currently that means @@ -385,6 +390,7 @@ class ExprFactory { friend class ParserMacroExprFactory; friend class OptimizerExprFactory; friend class tools::ProtoToPredicateBuilder; + friend class parser_internal::AstFactoryInterface; ExprFactory() : accu_var_(kAccumulatorVariableName) {} diff --git a/parser/internal/BUILD b/parser/internal/BUILD index c5bb23a4e..55d2aa247 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -20,6 +20,23 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) +cc_library( + name = "ast_factory_interface", + hdrs = ["ast_factory_interface.h"], +) + +cc_library( + name = "ast_factory", + hdrs = ["ast_factory.h"], + deps = [ + ":ast_factory_interface", + "//common:constant", + "//common:expr", + "//common:expr_factory", + "@com_google_absl//absl/strings:string_view", + ], +) + cc_library( name = "options", hdrs = ["options.h"], @@ -47,6 +64,17 @@ cc_library( ], ) +cc_test( + name = "ast_factory_test", + srcs = ["ast_factory_test.cc"], + deps = [ + ":ast_factory", + "//common:expr", + "//internal:testing", + "@com_google_absl//absl/strings:string_view", + ], +) + cc_test( name = "lexer_test", srcs = ["lexer_test.cc"], diff --git a/parser/internal/ast_factory.h b/parser/internal/ast_factory.h new file mode 100644 index 000000000..c0e8a6b7e --- /dev/null +++ b/parser/internal/ast_factory.h @@ -0,0 +1,181 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "parser/internal/ast_factory_interface.h" + +namespace cel::parser_internal { + +// Explicit specialization of `AstFactoryInterface` for `cel::Expr` AST nodes. + +template <> +class ListNodeBuilder { + public: + explicit ListNodeBuilder(int64_t id) { + expr_.set_id(id); + expr_.mutable_list_expr(); + } + + ListNodeBuilder& Add(cel::Expr element, bool optional = false) { + cel::ListExpr& list_val = expr_.mutable_list_expr(); + cel::ListExprElement expr_element; + expr_element.set_expr(std::move(element)); + expr_element.set_optional(optional); + list_val.mutable_elements().push_back(std::move(expr_element)); + return *this; + } + + cel::Expr Build() { return std::move(expr_); } + + private: + cel::Expr expr_; +}; + +template <> +class MapNodeBuilder { + public: + explicit MapNodeBuilder(int64_t id) { + expr_.set_id(id); + expr_.mutable_map_expr(); + } + + MapNodeBuilder& Add(int64_t id, cel::Expr key, cel::Expr value, + bool optional = false) { + cel::MapExpr& map_val = expr_.mutable_map_expr(); + cel::MapExprEntry entry; + entry.set_id(id); + entry.set_key(std::move(key)); + entry.set_value(std::move(value)); + entry.set_optional(optional); + map_val.mutable_entries().push_back(std::move(entry)); + return *this; + } + + cel::Expr Build() { return std::move(expr_); } + + private: + cel::Expr expr_; +}; + +template <> +class StructNodeBuilder { + public: + explicit StructNodeBuilder(int64_t id, std::string name) { + expr_.set_id(id); + expr_.mutable_struct_expr().set_name(std::move(name)); + } + + StructNodeBuilder& Add(int64_t id, std::string name, cel::Expr value, + bool optional = false) { + cel::StructExpr& struct_val = expr_.mutable_struct_expr(); + cel::StructExprField field; + field.set_id(id); + field.set_name(std::move(name)); + field.set_value(std::move(value)); + field.set_optional(optional); + struct_val.mutable_fields().push_back(std::move(field)); + return *this; + } + + cel::Expr Build() { return std::move(expr_); } + + private: + cel::Expr expr_; +}; + +template <> +class AstFactoryInterface : public cel::ExprFactory { + public: + AstFactoryInterface() = default; + AstFactoryInterface(const AstFactoryInterface&) = delete; + AstFactoryInterface(AstFactoryInterface&&) = delete; + AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; + AstFactoryInterface& operator=(AstFactoryInterface&&) = delete; + + ~AstFactoryInterface() override = default; + + // Node inspection and encapsulation API + int64_t GetId(const cel::Expr& expr) const { return expr.id(); } + + bool IsEmpty(const cel::Expr& expr) const { return expr.id() == 0; } + + bool IsConst(const cel::Expr& expr) const { return expr.has_const_expr(); } + + bool IsIdent(const cel::Expr& expr) const { return expr.has_ident_expr(); } + + absl::string_view GetIdentName(const cel::Expr& expr) const { + return expr.has_ident_expr() ? absl::string_view(expr.ident_expr().name()) + : absl::string_view(); + } + + bool IsSelect(const cel::Expr& expr) const { return expr.has_select_expr(); } + + bool IsPresenceTest(const cel::Expr& expr) const { + return expr.has_select_expr() && expr.select_expr().test_only(); + } + + const cel::Expr* GetSelectOperand(const cel::Expr& expr) const { + return expr.has_select_expr() ? &expr.select_expr().operand() : nullptr; + } + + absl::string_view GetSelectField(const cel::Expr& expr) const { + return expr.has_select_expr() + ? absl::string_view(expr.select_expr().field()) + : absl::string_view(); + } + + // Node creation API + using cel::ExprFactory::NewBoolConst; + using cel::ExprFactory::NewBytesConst; + using cel::ExprFactory::NewCall; + using cel::ExprFactory::NewDoubleConst; + using cel::ExprFactory::NewIdent; + using cel::ExprFactory::NewIntConst; + using cel::ExprFactory::NewMemberCall; + using cel::ExprFactory::NewNullConst; + using cel::ExprFactory::NewPresenceTest; + using cel::ExprFactory::NewSelect; + using cel::ExprFactory::NewStringConst; + using cel::ExprFactory::NewUintConst; + using cel::ExprFactory::NewUnspecified; + + ListNodeBuilder NewListBuilder(int64_t id) { + return ListNodeBuilder(id); + } + + StructNodeBuilder NewStructBuilder(int64_t id, std::string name) { + return StructNodeBuilder(id, std::move(name)); + } + + MapNodeBuilder NewMapBuilder(int64_t id) { + return MapNodeBuilder(id); + } +}; + +using AstFactory = AstFactoryInterface; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ diff --git a/parser/internal/ast_factory_interface.h b/parser/internal/ast_factory_interface.h new file mode 100644 index 000000000..3b8c9bdfb --- /dev/null +++ b/parser/internal/ast_factory_interface.h @@ -0,0 +1,104 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ + +#include +#include +#include +#include + +namespace cel::parser_internal { + +// Interface for decoupling parser logic from the underlying AST node +// data structures. +// +// By parameterizing the parser and factory on `ExprNode`, alternative AST node +// representations (such as `cel::Expr`) can be constructed without modifying +// parser rules. +// +// To implement AST construction using an alternative AST structure: +// 1. Define or specify your custom node type `MyNode`. +// 2. Implement a concrete factory specialization `AstFactoryInterface` +// that provides inspection (`GetId`, `IsSelect`, etc.) and creation +// (`NewCall`, `NewListBuilder`, etc.) operations for `MyNode`. +// 3. Instantiate the parser worker with your node type: +// `PrattParserWorker`. + +template +class ListNodeBuilder { + public: + ListNodeBuilder& Add(ExprNode element, bool optional = false); + ExprNode Build(); +}; + +template +class MapNodeBuilder { + public: + MapNodeBuilder& Add(int64_t id, ExprNode key, ExprNode value, + bool optional = false); + ExprNode Build(); +}; + +template +class StructNodeBuilder { + public: + StructNodeBuilder& Add(int64_t id, std::string name, ExprNode value, + bool optional = false); + ExprNode Build(); +}; + +template +class AstFactoryInterface { + public: + AstFactoryInterface() = default; + AstFactoryInterface(const AstFactoryInterface&) = delete; + AstFactoryInterface(AstFactoryInterface&&) = delete; + AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; + AstFactoryInterface& operator=(AstFactoryInterface&&) = delete; + + int64_t GetId(const ExprNode& expr) const; + bool IsEmpty(const ExprNode& expr) const; + bool IsConst(const ExprNode& expr) const; + bool IsIdent(const ExprNode& expr) const; + std::string_view GetIdentName(const ExprNode& expr) const; + bool IsSelect(const ExprNode& expr) const; + bool IsPresenceTest(const ExprNode& expr) const; + const ExprNode* GetSelectOperand(const ExprNode& expr) const; + std::string_view GetSelectField(const ExprNode& expr) const; + + ExprNode NewUnspecified(int64_t id); + ExprNode NewNullConst(int64_t id); + ExprNode NewBoolConst(int64_t id, bool value); + ExprNode NewIntConst(int64_t id, int64_t value); + ExprNode NewUintConst(int64_t id, uint64_t value); + ExprNode NewDoubleConst(int64_t id, double value); + ExprNode NewBytesConst(int64_t id, std::string value); + ExprNode NewStringConst(int64_t id, std::string value); + ExprNode NewIdent(int64_t id, std::string name); + ExprNode NewSelect(int64_t id, ExprNode operand, std::string field); + ExprNode NewPresenceTest(int64_t id, ExprNode operand, std::string field); + ExprNode NewCall(int64_t id, std::string function, + std::vector args); + ExprNode NewMemberCall(int64_t id, std::string function, ExprNode target, + std::vector args); + ListNodeBuilder NewListBuilder(int64_t id); + MapNodeBuilder NewMapBuilder(int64_t id); + StructNodeBuilder NewStructBuilder(int64_t id, std::string name); +}; + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ diff --git a/parser/internal/ast_factory_test.cc b/parser/internal/ast_factory_test.cc new file mode 100644 index 000000000..a66c8ee6b --- /dev/null +++ b/parser/internal/ast_factory_test.cc @@ -0,0 +1,200 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/ast_factory.h" + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "common/expr.h" +#include "internal/testing.h" + +namespace cel::parser_internal { +namespace { + +TEST(AstFactoryInterfaceTest, AstFactoryUnspecified) { + AstFactory factory; + cel::Expr expr = factory.NewUnspecified(1); + EXPECT_EQ(factory.GetId(expr), 1); + EXPECT_FALSE(factory.IsEmpty(expr)); + EXPECT_FALSE(factory.IsConst(expr)); + EXPECT_FALSE(factory.IsIdent(expr)); + EXPECT_FALSE(factory.IsSelect(expr)); + + cel::Expr empty_expr = factory.NewUnspecified(0); + EXPECT_TRUE(factory.IsEmpty(empty_expr)); +} + +TEST(AstFactoryInterfaceTest, AstFactoryConstNodes) { + AstFactory factory; + + cel::Expr null_expr = factory.NewNullConst(10); + EXPECT_EQ(factory.GetId(null_expr), 10); + EXPECT_TRUE(factory.IsConst(null_expr)); + ASSERT_TRUE(null_expr.has_const_expr()); + EXPECT_TRUE(null_expr.const_expr().has_null_value()); + + cel::Expr bool_expr = factory.NewBoolConst(11, true); + EXPECT_EQ(factory.GetId(bool_expr), 11); + EXPECT_TRUE(factory.IsConst(bool_expr)); + ASSERT_TRUE(bool_expr.has_const_expr()); + EXPECT_TRUE(bool_expr.const_expr().bool_value()); + + cel::Expr int_expr = factory.NewIntConst(12, -42); + EXPECT_EQ(factory.GetId(int_expr), 12); + EXPECT_TRUE(factory.IsConst(int_expr)); + ASSERT_TRUE(int_expr.has_const_expr()); + EXPECT_EQ(int_expr.const_expr().int_value(), -42); + + cel::Expr uint_expr = factory.NewUintConst(13, 100u); + EXPECT_EQ(factory.GetId(uint_expr), 13); + EXPECT_TRUE(factory.IsConst(uint_expr)); + ASSERT_TRUE(uint_expr.has_const_expr()); + EXPECT_EQ(uint_expr.const_expr().uint_value(), 100u); + + cel::Expr double_expr = factory.NewDoubleConst(14, 3.14159); + EXPECT_EQ(factory.GetId(double_expr), 14); + EXPECT_TRUE(factory.IsConst(double_expr)); + ASSERT_TRUE(double_expr.has_const_expr()); + EXPECT_DOUBLE_EQ(double_expr.const_expr().double_value(), 3.14159); + + cel::Expr bytes_expr = factory.NewBytesConst(15, "bytes_val"); + EXPECT_EQ(factory.GetId(bytes_expr), 15); + EXPECT_TRUE(factory.IsConst(bytes_expr)); + ASSERT_TRUE(bytes_expr.has_const_expr()); + EXPECT_EQ(bytes_expr.const_expr().bytes_value(), "bytes_val"); + + cel::Expr string_expr = factory.NewStringConst(16, "string_val"); + EXPECT_EQ(factory.GetId(string_expr), 16); + EXPECT_TRUE(factory.IsConst(string_expr)); + ASSERT_TRUE(string_expr.has_const_expr()); + EXPECT_EQ(string_expr.const_expr().string_value(), "string_val"); +} + +TEST(AstFactoryInterfaceTest, AstFactoryIdentAndSelect) { + AstFactory factory; + + cel::Expr ident_expr = factory.NewIdent(20, "foo"); + EXPECT_EQ(factory.GetId(ident_expr), 20); + EXPECT_TRUE(factory.IsIdent(ident_expr)); + EXPECT_EQ(factory.GetIdentName(ident_expr), "foo"); + + cel::Expr select_expr = + factory.NewSelect(21, factory.NewIdent(20, "foo"), "bar"); + EXPECT_EQ(factory.GetId(select_expr), 21); + EXPECT_TRUE(factory.IsSelect(select_expr)); + EXPECT_FALSE(factory.IsPresenceTest(select_expr)); + EXPECT_EQ(factory.GetSelectField(select_expr), "bar"); + ASSERT_NE(factory.GetSelectOperand(select_expr), nullptr); + EXPECT_EQ(factory.GetIdentName(*factory.GetSelectOperand(select_expr)), + "foo"); + + cel::Expr presence_expr = + factory.NewPresenceTest(22, factory.NewIdent(20, "foo"), "bar"); + EXPECT_EQ(factory.GetId(presence_expr), 22); + EXPECT_TRUE(factory.IsSelect(presence_expr)); + EXPECT_TRUE(factory.IsPresenceTest(presence_expr)); + EXPECT_EQ(factory.GetSelectField(presence_expr), "bar"); +} + +TEST(AstFactoryInterfaceTest, AstFactoryCalls) { + AstFactory factory; + + std::vector call_args; + call_args.push_back(factory.NewIntConst(30, 1)); + call_args.push_back(factory.NewIntConst(31, 2)); + cel::Expr call_expr = factory.NewCall(32, "_+_", std::move(call_args)); + EXPECT_EQ(factory.GetId(call_expr), 32); + ASSERT_TRUE(call_expr.has_call_expr()); + EXPECT_EQ(call_expr.call_expr().function(), "_+_"); + EXPECT_FALSE(call_expr.call_expr().has_target()); + EXPECT_EQ(call_expr.call_expr().args().size(), 2); + + std::vector member_args; + member_args.push_back(factory.NewStringConst(33, "suffix")); + cel::Expr member_call_expr = factory.NewMemberCall( + 34, "endsWith", factory.NewIdent(35, "str_var"), std::move(member_args)); + EXPECT_EQ(factory.GetId(member_call_expr), 34); + ASSERT_TRUE(member_call_expr.has_call_expr()); + EXPECT_EQ(member_call_expr.call_expr().function(), "endsWith"); + EXPECT_TRUE(member_call_expr.call_expr().has_target()); + EXPECT_EQ(member_call_expr.call_expr().target().ident_expr().name(), + "str_var"); + EXPECT_EQ(member_call_expr.call_expr().args().size(), 1); +} + +TEST(AstFactoryInterfaceTest, AstFactoryList) { + AstFactory factory; + + cel::Expr list_expr = factory.NewListBuilder(42) + .Add(factory.NewIntConst(40, 1), false) + .Add(factory.NewIntConst(41, 2), true) + .Build(); + EXPECT_EQ(factory.GetId(list_expr), 42); + ASSERT_TRUE(list_expr.has_list_expr()); + ASSERT_EQ(list_expr.list_expr().elements().size(), 2); + EXPECT_FALSE(list_expr.list_expr().elements()[0].optional()); + EXPECT_EQ(list_expr.list_expr().elements()[0].expr().const_expr().int_value(), + 1); + EXPECT_TRUE(list_expr.list_expr().elements()[1].optional()); + EXPECT_EQ(list_expr.list_expr().elements()[1].expr().const_expr().int_value(), + 2); +} + +TEST(AstFactoryInterfaceTest, AstFactoryStruct) { + AstFactory factory; + + cel::Expr struct_expr = + factory.NewStructBuilder(54, "MyMessage") + .Add(50, "field1", factory.NewIntConst(51, 100), false) + .Add(52, "field2", factory.NewIntConst(53, 200), true) + .Build(); + EXPECT_EQ(factory.GetId(struct_expr), 54); + ASSERT_TRUE(struct_expr.has_struct_expr()); + EXPECT_EQ(struct_expr.struct_expr().name(), "MyMessage"); + ASSERT_EQ(struct_expr.struct_expr().fields().size(), 2); + EXPECT_EQ(struct_expr.struct_expr().fields()[0].id(), 50); + EXPECT_EQ(struct_expr.struct_expr().fields()[0].name(), "field1"); + EXPECT_FALSE(struct_expr.struct_expr().fields()[0].optional()); + EXPECT_EQ(struct_expr.struct_expr().fields()[1].id(), 52); + EXPECT_EQ(struct_expr.struct_expr().fields()[1].name(), "field2"); + EXPECT_TRUE(struct_expr.struct_expr().fields()[1].optional()); +} + +TEST(AstFactoryInterfaceTest, AstFactoryMap) { + AstFactory factory; + + cel::Expr map_expr = factory.NewMapBuilder(66) + .Add(60, factory.NewStringConst(61, "key1"), + factory.NewIntConst(62, 10), false) + .Add(63, factory.NewStringConst(64, "key2"), + factory.NewIntConst(65, 20), true) + .Build(); + EXPECT_EQ(factory.GetId(map_expr), 66); + ASSERT_TRUE(map_expr.has_map_expr()); + ASSERT_EQ(map_expr.map_expr().entries().size(), 2); + EXPECT_EQ(map_expr.map_expr().entries()[0].id(), 60); + EXPECT_EQ(map_expr.map_expr().entries()[0].key().const_expr().string_value(), + "key1"); + EXPECT_EQ(map_expr.map_expr().entries()[0].value().const_expr().int_value(), + 10); + EXPECT_FALSE(map_expr.map_expr().entries()[0].optional()); + EXPECT_EQ(map_expr.map_expr().entries()[1].id(), 63); + EXPECT_TRUE(map_expr.map_expr().entries()[1].optional()); +} + +} // namespace +} // namespace cel::parser_internal From 781ecf17810fa2ceda6ee45de34b9762bf5fa2fc Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 23 Jul 2026 17:28:31 -0700 Subject: [PATCH 132/350] [Pratt Parser] Add PrattParser for core grammar. Macro expansion is not implemented yet PiperOrigin-RevId: 953044015 --- parser/internal/BUILD | 76 ++ parser/internal/pratt_parser.cc | 217 ++++ parser/internal/pratt_parser.h | 72 ++ parser/internal/pratt_parser_test.cc | 1434 ++++++++++++++++++++++++ parser/internal/pratt_parser_worker.cc | 158 +++ parser/internal/pratt_parser_worker.h | 860 ++++++++++++++ 6 files changed, 2817 insertions(+) create mode 100644 parser/internal/pratt_parser.cc create mode 100644 parser/internal/pratt_parser.h create mode 100644 parser/internal/pratt_parser_test.cc create mode 100644 parser/internal/pratt_parser_worker.cc create mode 100644 parser/internal/pratt_parser_worker.h diff --git a/parser/internal/BUILD b/parser/internal/BUILD index 55d2aa247..0d2af99d2 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -64,6 +64,56 @@ cc_library( ], ) +cc_library( + name = "pratt_parser_worker", + srcs = ["pratt_parser_worker.cc"], + hdrs = ["pratt_parser_worker.h"], + deps = [ + ":ast_factory_interface", + ":lexer", + "//common:operators", + "//common:source", + "//internal:lexis", + "//internal:strings", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "pratt_parser", + srcs = ["pratt_parser.cc"], + hdrs = ["pratt_parser.h"], + deps = [ + ":ast_factory", + ":pratt_parser_worker", + "//common:ast", + "//common:expr", + "//common:source", + "//internal:status_macros", + "//parser:macro", + "//parser:macro_registry", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:span", + ], +) + cc_test( name = "ast_factory_test", srcs = ["ast_factory_test.cc"], @@ -84,3 +134,29 @@ cc_test( "//internal:testing", ], ) + +cc_test( + name = "pratt_parser_test", + srcs = ["pratt_parser_test.cc"], + deps = [ + ":lexer", + ":pratt_parser", + ":pratt_parser_worker", + "//common:ast", + "//common:constant", + "//common:expr", + "//common:source", + "//internal:status_macros", + "//internal:testing", + "//parser:options", + "//parser:parser_interface", + "//testutil:expr_printer", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc new file mode 100644 index 000000000..f86974a36 --- /dev/null +++ b/parser/internal/pratt_parser.cc @@ -0,0 +1,217 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast.h" +#include "common/expr.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "parser/internal/ast_factory.h" // IWYU pragma: keep +#include "parser/internal/pratt_parser_worker.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +namespace { + +std::string DisplayParserError(const cel::Source& source, + SourceLocation location, + std::string_view message) { + return absl::StrCat( + absl::StrFormat("ERROR: %s:%zu:%zu: %s", source.description(), + location.line, location.column + 1, message), + source.DisplayErrorLocation(location)); +} + +std::string FormatIssues(const cel::Source& source, + absl::Span issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, DisplayParserError(source, issue.location(), issue.message())); + }); +} + +class PrattParserBuilderImpl final : public cel::ParserBuilder { + public: + explicit PrattParserBuilderImpl(const cel::ParserOptions& options) + : options_(options) {} + + cel::ParserOptions& GetOptions() override { return options_; } + + absl::Status AddMacro(const cel::Macro& macro) override { + for (const cel::Macro& existing_macro : macros_) { + if (existing_macro.key() == macro.key()) { + return absl::AlreadyExistsError( + absl::StrCat("macro already exists: ", macro.key())); + } + } + macros_.push_back(macro); + return absl::OkStatus(); + } + + absl::Status AddLibrary(cel::ParserLibrary library) override { + if (!library.id.empty()) { + auto [it, inserted] = library_ids_.insert(library.id); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library already exists: ", library.id)); + } + } + libraries_.push_back(std::move(library)); + return absl::OkStatus(); + } + + absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override { + if (subset.library_id.empty()) { + return absl::InvalidArgumentError("subset must have a library id"); + } + std::string library_id = subset.library_id; + auto [it, inserted] = + library_subsets_.insert({library_id, std::move(subset)}); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library subset already exists: ", library_id)); + } + return absl::OkStatus(); + } + + absl::StatusOr> Build() override { + using std::swap; + std::vector individual_macros; + swap(individual_macros, macros_); + absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); + + cel::MacroRegistry macro_registry; + + for (const cel::ParserLibrary& library : libraries_) { + CEL_RETURN_IF_ERROR(library.configure(*this)); + if (!library.id.empty()) { + auto it = library_subsets_.find(library.id); + if (it != library_subsets_.end()) { + const cel::ParserLibrarySubset& subset = it->second; + for (const cel::Macro& macro : macros_) { + if (subset.should_include_macro(macro)) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); + } + } + macros_.clear(); + continue; + } + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); + macros_.clear(); + } + + absl::flat_hash_set library_ids(library_ids_); + + if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); + library_ids.insert("stdlib"); + } + + if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); + library_ids.insert("optional"); + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); + return std::make_unique( + options_, std::move(macro_registry), std::move(library_ids)); + } + + cel::ParserOptions options_; + std::vector macros_; + absl::flat_hash_set library_ids_; + std::vector libraries_; + absl::flat_hash_map library_subsets_; +}; + +} // namespace + +template class PrattParserWorker; + +absl::StatusOr> PrattParserImpl::ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const { + if (source.content().size() > options_.expression_size_codepoint_limit) { + return absl::InvalidArgumentError(absl::StrFormat( + "expression size exceeds codepoint limit: %zu > %d", + source.content().size(), options_.expression_size_codepoint_limit)); + } + std::vector issues; + PrattParserWorker worker(source, options_, &issues); + Expr expr = worker.Parse(); + if (worker.is_recursion_limit_exceeded()) { + return absl::CancelledError( + absl::StrFormat("Expression recursion limit exceeded. limit: %d", + options_.max_recursion_depth)); + } + if (worker.has_errors()) { + std::string err_msg = FormatIssues(source, issues); + if (parse_issues != nullptr) { + parse_issues->swap(issues); + } + return absl::InvalidArgumentError(err_msg); + } + + cel::SourceInfo source_info; + source_info.set_location(std::string(source.description())); + for (const auto& [id, pos] : worker.GetNodePositions()) { + source_info.mutable_positions().insert({id, pos}); + } + source_info.mutable_line_offsets().reserve(worker.GetLineOffsets().size()); + for (int32_t offset : worker.GetLineOffsets()) { + source_info.mutable_line_offsets().push_back(offset); + } + return std::make_unique(std::move(expr), std::move(source_info)); +} + +std::unique_ptr PrattParserImpl::ToBuilder() const { + auto ins = std::make_unique(options_); + ins->library_ids_ = library_ids_; + ins->macros_ = macro_registry_.ListMacros(); + return ins; +} + +std::unique_ptr NewPrattParserBuilder( + const cel::ParserOptions& options) { + return std::make_unique(options); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser.h b/parser/internal/pratt_parser.h new file mode 100644 index 000000000..c39c68521 --- /dev/null +++ b/parser/internal/pratt_parser.h @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" +#include "common/ast.h" +#include "common/source.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +// PrattParserImpl implements the Pratt parsing algorithm for CEL expressions. +// +// WARNING: Since this implementation uses recursive descent to parse +// expressions, its stack consumption depends on expression nesting and +// recursion limits (`ParserOptions::max_recursion_depth`). Note that: +// In production builds (e.g., inside fibers with small default stacks such as +// 64KB), the available stack space may be much tighter than in default thread +// stacks. +// Consequently, `ParserOptions::max_recursion_depth` may need to be tuned +// depending on the caller environment to prevent stack overflow. +class PrattParserImpl final : public cel::Parser { + public: + explicit PrattParserImpl(const cel::ParserOptions& options, + cel::MacroRegistry macro_registry, + absl::flat_hash_set library_ids) + : options_(options), + macro_registry_(std::move(macro_registry)), + library_ids_(std::move(library_ids)) {} + + ~PrattParserImpl() override = default; + + absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const override; + + std::unique_ptr ToBuilder() const override; + + private: + cel::ParserOptions options_; + cel::MacroRegistry macro_registry_; + absl::flat_hash_set library_ids_; +}; + +std::unique_ptr NewPrattParserBuilder( + const cel::ParserOptions& options = cel::ParserOptions()); + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_H_ diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc new file mode 100644 index 000000000..b1dd81fbb --- /dev/null +++ b/parser/internal/pratt_parser_test.cc @@ -0,0 +1,1434 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser.h" + +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "parser/internal/lexer.h" +#include "parser/internal/pratt_parser_worker.h" +#include "parser/options.h" +#include "parser/parser_interface.h" +#include "testutil/expr_printer.h" + +// Change to 0 to test with the ANTLR parser to check for differences. +#define USE_PRATT_PARSER 1 + +namespace cel::parser_internal { +namespace { + +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::testing::Eq; +using ::testing::NotNull; + +absl::StatusOr> Parse( + std::string_view expression, + const cel::ParserOptions& options = cel::ParserOptions(), + std::vector* issues = nullptr) { +#if USE_PRATT_PARSER + std::unique_ptr builder = NewPrattParserBuilder(options); +#else + std::unique_ptr builder = cel::NewParserBuilder(options); +#endif + CEL_ASSIGN_OR_RETURN(std::unique_ptr parser, builder->Build()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr source, + cel::NewSource(expression)); + return parser->Parse(*source, issues); +} + +struct TestCase { + std::string_view source; + std::string_view expected_ast; + bool enable_optional_syntax = false; + bool enable_variadic_logical_operators = false; +}; + +class PrattParserTest : public testing::TestWithParam {}; + +std::string Unindent(std::string_view multiline) { + std::vector unindented_lines; + int indent = -1; + for (std::string_view line : absl::StrSplit(multiline, '\n')) { + std::size_t pos = line.find_first_not_of(" \t"); + if (pos == std::string_view::npos) continue; + if (indent == -1) indent = pos; + unindented_lines.push_back(std::string(line.substr(indent))); + } + return absl::StrJoin(unindented_lines, "\n"); +} + +std::string_view ConstantKind(const cel::Constant& c) { + switch (c.kind_case()) { + case ConstantKindCase::kBool: + return "bool"; + case ConstantKindCase::kInt: + return "int64"; + case ConstantKindCase::kUint: + return "uint64"; + case ConstantKindCase::kDouble: + return "double"; + case ConstantKindCase::kString: + return "string"; + case ConstantKindCase::kBytes: + return "bytes"; + case ConstantKindCase::kNull: + return "NullValue"; + default: + return "unspecified_constant"; + } +} + +std::string_view ExprKind(const cel::Expr& e) { + switch (e.kind_case()) { + case ExprKindCase::kConstant: + // special cased, this doesn't appear. + return "Expr.Constant"; + case ExprKindCase::kIdentExpr: + return "Expr.Ident"; + case ExprKindCase::kSelectExpr: + return "Expr.Select"; + case ExprKindCase::kCallExpr: + return "Expr.Call"; + case ExprKindCase::kListExpr: + return "Expr.CreateList"; + case ExprKindCase::kMapExpr: + return "Expr.CreateMap"; + case ExprKindCase::kStructExpr: + return "Expr.CreateStruct"; + case ExprKindCase::kComprehensionExpr: + return "Expr.Comprehension"; + default: + return "unspecified_expr"; + } +} + +class KindAndIdAdorner : public cel::test::ExpressionAdorner { + public: + std::string Adorn(const cel::Expr& e) const override { + if (e.has_const_expr()) { + const cel::Constant& const_expr = e.const_expr(); + return absl::StrCat("^#", e.id(), ":", ConstantKind(const_expr), "#"); + } else { + return absl::StrCat("^#", e.id(), ":", ExprKind(e), "#"); + } + } + + std::string AdornStructField(const cel::StructExprField& e) const override { + return absl::StrFormat("^#%d:Expr.CreateStruct.Entry#", e.id()); + } + + std::string AdornMapEntry(const cel::MapExprEntry& e) const override { + return absl::StrFormat("^#%d:Expr.CreateStruct.Entry#", e.id()); + } +}; + +TEST_P(PrattParserTest, Parse) { + const TestCase& test_case = GetParam(); + cel::ParserOptions options; + options.enable_optional_syntax = test_case.enable_optional_syntax; + options.enable_variadic_logical_operators = + test_case.enable_variadic_logical_operators; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, + Parse(test_case.source, options)); + const Expr& root = ast->root_expr(); + KindAndIdAdorner kind_and_id_adorner; + test::ExprPrinter printer(kind_and_id_adorner); + EXPECT_EQ(Unindent(printer.Print(root)), Unindent(test_case.expected_ast)); +} + +std::vector GetParserTestCases() { + return { + TestCase{ + .source = "null", + .expected_ast = R"( + null^#1:NullValue# + )", + }, + TestCase{ + .source = "true", + .expected_ast = R"( + true^#1:bool# + )", + }, + TestCase{ + .source = "false", + .expected_ast = R"( + false^#1:bool# + )", + }, + TestCase{ + .source = "123", + .expected_ast = R"( + 123^#1:int64# + )", + }, + TestCase{ + .source = "-123", + .expected_ast = R"( + -123^#1:int64# + )", + }, + TestCase{ + .source = "-9223372036854775808", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "0xA", + .expected_ast = R"( + 10^#1:int64# + )", + }, + TestCase{ + .source = "-0x1A", + .expected_ast = R"( + -26^#1:int64# + )", + }, + TestCase{ + .source = "-0X1a", + .expected_ast = R"( + -26^#1:int64# + )", + }, + TestCase{ + .source = "-0x8000000000000000", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "-0X8000000000000000", + .expected_ast = R"( + -9223372036854775808^#1:int64# + )", + }, + TestCase{ + .source = "42u", + .expected_ast = R"( + 42u^#1:uint64# + )", + }, + TestCase{ + .source = "0u", + .expected_ast = R"( + 0u^#1:uint64# + )", + }, + TestCase{ + .source = "0xAu", + .expected_ast = R"( + 10u^#1:uint64# + )", + }, + TestCase{ + .source = "3.14159", + .expected_ast = R"( + 3.14159^#1:double# + )", + }, + TestCase{ + .source = "-3.14159", + .expected_ast = R"( + -3.14159^#1:double# + )", + }, + TestCase{ + .source = "-5.5e-3", + .expected_ast = R"( + -0.0055^#1:double# + )", + }, + TestCase{ + .source = "b'hello'", + .expected_ast = R"( + b"hello"^#1:bytes# + )", + }, + TestCase{ + .source = "b\"hello\"", + .expected_ast = R"( + b"hello"^#1:bytes# + )", + }, + TestCase{ + .source = "'hello world'", + .expected_ast = R"( + "hello world"^#1:string# + )", + }, + TestCase{ + .source = "\"hello world\"", + .expected_ast = R"( + "hello world"^#1:string# + )", + }, + TestCase{ + .source = "\"\u2764\"", + .expected_ast = "\"\u2764\"^#1:string#", + }, + TestCase{ + .source = "\"\\a\\b\\f\\n\\r\\t\\v'\\\"\\\\\\? Legal escapes\"", + .expected_ast = R"( + "\x07\x08\x0c\n\r\t\x0b'\"\\? Legal escapes"^#1:string# + )", + }, + TestCase{ + .source = "a", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "(a)", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "((a))", + .expected_ast = R"( + a^#1:Expr.Ident# + )", + }, + TestCase{ + .source = "a.b", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.?b", + .expected_ast = R"( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "a.b.c", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select#.c^#3:Expr.Select# + )", + }, + TestCase{ + .source = "a.`b`", + .expected_ast = R"( + a^#1:Expr.Ident#.b^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.`b-c`", + .expected_ast = R"( + a^#1:Expr.Ident#.b-c^#2:Expr.Select# + )", + }, + TestCase{ + .source = "a.?`b`", + .expected_ast = R"( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "-x", + .expected_ast = R"( + -_( + x^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "- -1", + .expected_ast = R"( + -_( + -1^#2:int64# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-(1 + 2)", + .expected_ast = R"( + -_( + _+_( + 1^#2:int64#, + 2^#4:int64# + )^#3:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "---a", + .expected_ast = R"( + -_( + -_( + -_( + a^#4:Expr.Ident# + )^#3:Expr.Call# + )^#2:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!false", + .expected_ast = R"( + !_( + false^#2:bool# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!a", + .expected_ast = R"( + !_( + a^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-!true", + .expected_ast = R"( + -_( + !_( + true^#3:bool# + )^#2:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a + b", + .expected_ast = R"( + _+_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a - b", + .expected_ast = R"( + _-_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "4--4", + .expected_ast = R"( + _-_( + 4^#1:int64#, + -4^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a * b", + .expected_ast = R"( + _*_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a / b", + .expected_ast = R"( + _/_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a % b", + .expected_ast = R"( + _%_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a in b", + .expected_ast = R"( + @in( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a == b", + .expected_ast = R"( + _==_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a != b", + .expected_ast = R"( + _!=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a > b", + .expected_ast = R"( + _>_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a >= b", + .expected_ast = R"( + _>=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a < b", + .expected_ast = R"( + _<_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a <= b", + .expected_ast = R"( + _<=_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a && b", + .expected_ast = R"( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c", + .expected_ast = R"( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d", + .expected_ast = R"( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + _&&_( + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#7:Expr.Call# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d && e", + .expected_ast = R"( + _&&_( + _&&_( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call#, + _&&_( + d^#6:Expr.Ident#, + e^#8:Expr.Ident# + )^#9:Expr.Call# + )^#7:Expr.Call# + )", + }, + TestCase{ + .source = "a && b && c && d", + .expected_ast = R"( + _&&_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident#, + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#3:Expr.Call# + )", + .enable_variadic_logical_operators = true, + }, + TestCase{ + .source = "a || b", + .expected_ast = R"( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c", + .expected_ast = R"( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d", + .expected_ast = R"( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + _||_( + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#7:Expr.Call# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d || e", + .expected_ast = R"( + _||_( + _||_( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident# + )^#3:Expr.Call#, + c^#4:Expr.Ident# + )^#5:Expr.Call#, + _||_( + d^#6:Expr.Ident#, + e^#8:Expr.Ident# + )^#9:Expr.Call# + )^#7:Expr.Call# + )", + }, + TestCase{ + .source = "a || b || c || d", + .expected_ast = R"( + _||_( + a^#1:Expr.Ident#, + b^#2:Expr.Ident#, + c^#4:Expr.Ident#, + d^#6:Expr.Ident# + )^#3:Expr.Call# + )", + .enable_variadic_logical_operators = true, + }, + TestCase{ + .source = "10 - 3 - 2", + .expected_ast = R"( + _-_( + _-_( + 10^#1:int64#, + 3^#3:int64# + )^#2:Expr.Call#, + 2^#5:int64# + )^#4:Expr.Call# + )", + }, + TestCase{ + .source = "1 + 2 * 3 - 1 / 2 == 6 % 1", + .expected_ast = R"( + _==_( + _-_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + _/_( + 1^#7:int64#, + 2^#9:int64# + )^#8:Expr.Call# + )^#6:Expr.Call#, + _%_( + 6^#11:int64#, + 1^#13:int64# + )^#12:Expr.Call# + )^#10:Expr.Call# + )", + }, + TestCase{ + .source = "1 + 2 * 3 == 7 && true || false", + .expected_ast = R"( + _||_( + _&&_( + _==_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + 7^#7:int64# + )^#6:Expr.Call#, + true^#8:bool# + )^#9:Expr.Call#, + false^#10:bool# + )^#11:Expr.Call# + )", + }, + TestCase{ + .source = "x > 0 ? 'pos' : 'neg'", + .expected_ast = R"( + _?_:_( + _>_( + x^#1:Expr.Ident#, + 0^#3:int64# + )^#2:Expr.Call#, + "pos"^#5:string#, + "neg"^#6:string# + )^#4:Expr.Call# + )", + }, + TestCase{ + .source = "a?b:c", + .expected_ast = R"( + _?_:_( + a^#1:Expr.Ident#, + b^#3:Expr.Ident#, + c^#4:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a[b]", + .expected_ast = R"( + _[_]( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a[?b]", + .expected_ast = R"( + _[?_]( + a^#1:Expr.Ident#, + b^#3:Expr.Ident# + )^#2:Expr.Call# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "a[3]", + .expected_ast = R"( + _[_]( + a^#1:Expr.Ident#, + 3^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "[1,3,4][0]", + .expected_ast = R"( + _[_]( + [ + 1^#2:int64#, + 3^#3:int64#, + 4^#4:int64# + ]^#1:Expr.CreateList#, + 0^#6:int64# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "a()", + .expected_ast = R"( + a()^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a(b)", + .expected_ast = R"( + a( + b^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a(b, c)", + .expected_ast = R"( + a( + b^#2:Expr.Ident#, + c^#3:Expr.Ident# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "a.b()", + .expected_ast = R"( + a^#1:Expr.Ident#.b()^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a.b(1)", + .expected_ast = R"( + a^#1:Expr.Ident#.b( + 1^#3:int64# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a.b(c)", + .expected_ast = R"( + a^#1:Expr.Ident#.b( + c^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "size(x) == x.size()", + .expected_ast = R"( + _==_( + size( + x^#2:Expr.Ident# + )^#1:Expr.Call#, + x^#4:Expr.Ident#.size()^#5:Expr.Call# + )^#3:Expr.Call# + )", + }, + TestCase{ + .source = "\"foo\".size()", + .expected_ast = R"( + "foo"^#1:string#.size()^#2:Expr.Call# + )", + }, + TestCase{ + .source = "[true, true].size() == 2", + .expected_ast = R"( + _==_( + [ + true^#2:bool#, + true^#3:bool# + ]^#1:Expr.CreateList#.size()^#4:Expr.Call#, + 2^#6:int64# + )^#5:Expr.Call# + )", + }, + TestCase{ + .source = "[]", + .expected_ast = R"( + []^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[a]", + .expected_ast = R"( + [ + a^#2:Expr.Ident# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[1, 2, 3]", + .expected_ast = R"( + [ + 1^#2:int64#, + 2^#3:int64#, + 3^#4:int64# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[1, 2, 3,]", + .expected_ast = R"( + [ + 1^#2:int64#, + 2^#3:int64#, + 3^#4:int64# + ]^#1:Expr.CreateList# + )", + }, + TestCase{ + .source = "[?1, 2]", + .expected_ast = R"( + [ + ?1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "{}", + .expected_ast = R"( + {}^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "{'key': 'value', 'num': 42}", + .expected_ast = R"( + { + "key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, + "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "{?'key': 'value', 'num': 42}", + .expected_ast = R"( + { + ?"key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, + "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + .enable_optional_syntax = true, + }, + TestCase{ + .source = "{foo: 5, bar: \"xyz\"}", + .expected_ast = R"( + { + foo^#2:Expr.Ident#:5^#4:int64#^#3:Expr.CreateStruct.Entry#, + bar^#5:Expr.Ident#:"xyz"^#7:string#^#6:Expr.CreateStruct.Entry# + }^#1:Expr.CreateMap# + )", + }, + TestCase{ + .source = "google.protobuf.Empty{}", + .expected_ast = R"( + google.protobuf.Empty{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "foo{ }", + .expected_ast = R"( + foo{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "foo{ a:b }", + .expected_ast = R"( + foo{ + a:b^#3:Expr.Ident#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "A{`b`: 1}", + .expected_ast = R"( + A{ + b:1^#3:int64#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "A{`b-c`: 1}", + .expected_ast = R"( + A{ + b-c:1^#3:int64#^#2:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Msg{field: 10, other: 'val'}", + .expected_ast = R"( + Msg{ + field:10^#3:int64#^#2:Expr.CreateStruct.Entry#, + other:"val"^#5:string#^#4:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Msg{?field: 10, other: 'val'}", + .expected_ast = R"( + Msg{ + ?field:10^#3:int64#^#2:Expr.CreateStruct.Entry#, + other:"val"^#5:string#^#4:Expr.CreateStruct.Entry# + }^#1:Expr.CreateStruct# + )", + .enable_optional_syntax = true, + }, + }; +} + +std::string TestName(const testing::TestParamInfo& test_info) { + std::string name = absl::StrCat(test_info.index, "-", test_info.param.source); + absl::c_replace_if(name, [](char c) { return !absl::ascii_isalnum(c); }, '_'); + return name; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserTest, PrattParserTest, + testing::ValuesIn(GetParserTestCases()), TestName); + +struct ErrorTestCase { + std::string_view source; + std::string_view expected_error; + bool enable_optional_syntax = false; + bool enable_quoted_identifiers = false; +}; + +class PrattParserErrorTest : public testing::TestWithParam {}; + +std::string FormatIssues(const cel::Source& source, + const std::vector& issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, + absl::StrFormat("ERROR: %s:%zu:%zu: %s", source.description(), + issue.location().line, issue.location().column + 1, + issue.message()), + source.DisplayErrorLocation(issue.location())); + }); +} + +TEST_P(PrattParserErrorTest, ParseSyntaxError) { + const ErrorTestCase& test_case = GetParam(); + cel::ParserOptions options; + options.enable_optional_syntax = test_case.enable_optional_syntax; + options.enable_quoted_identifiers = test_case.enable_quoted_identifiers; + std::vector issues; + absl::StatusOr> result = + Parse(test_case.source, options, &issues); + EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument, + Eq(test_case.expected_error))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr source, + cel::NewSource(test_case.source)); + EXPECT_EQ(FormatIssues(*source, issues), test_case.expected_error); +} + +std::vector GetErrorTestCases() { + return { + ErrorTestCase{ + .source = "1 + 2 * 3 4", + .expected_error = + "ERROR: :1:11: unexpected token after expression\n" + " | 1 + 2 * 3 4\n" + " | ..........^", + }, + ErrorTestCase{ + .source = "1{}", + .expected_error = + "ERROR: :1:2: unexpected token after expression\n" + " | 1{}\n" + " | .^", + }, + ErrorTestCase{ + .source = "true ? 1", + .expected_error = + "ERROR: :1:9: expected ':' in conditional expression\n" + " | true ? 1\n" + " | ........^", + }, + ErrorTestCase{ + .source = "a.?b", + .expected_error = "ERROR: :1:2: unsupported syntax '.?'\n" + " | a.?b\n" + " | .^", + }, + ErrorTestCase{ + .source = "a.", + .expected_error = + "ERROR: :1:3: expected identifier after '.'\n" + " | a.\n" + " | ..^", + }, + ErrorTestCase{ + .source = "a[?0]", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | a[?0]\n" + " | .^", + }, + ErrorTestCase{ + .source = ". *", + .expected_error = "ERROR: :1:3: expected identifier\n" + " | . *\n" + " | ..^", + }, + ErrorTestCase{ + .source = ".as", + .expected_error = "ERROR: :1:2: reserved identifier: as\n" + " | .as\n" + " | .^", + }, + ErrorTestCase{ + .source = "* 2", + .expected_error = + "ERROR: :1:1: unexpected token\n" + " | * 2\n" + " | ^\n" + "ERROR: :1:3: unexpected token after expression\n" + " | * 2\n" + " | ..^", + }, + ErrorTestCase{ + .source = "(1 + 2", + .expected_error = + "ERROR: :1:7: mismatched input expecting ')'\n" + " | (1 + 2\n" + " | ......^", + }, + ErrorTestCase{ + .source = "[?1]", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | [?1]\n" + " | .^", + }, + ErrorTestCase{ + .source = "[1, 2", + .expected_error = "ERROR: :1:6: expected ']'\n" + " | [1, 2\n" + " | .....^", + }, + ErrorTestCase{ + .source = "{?'k': 'v'}", + .expected_error = "ERROR: :1:2: unsupported syntax '?'\n" + " | {?'k': 'v'}\n" + " | .^", + }, + ErrorTestCase{ + .source = "{'k' 'v'}", + .expected_error = "ERROR: :1:6: expected ':' in map entry\n" + " | {'k' 'v'}\n" + " | .....^", + }, + ErrorTestCase{ + .source = "{'k': 'v'", + .expected_error = "ERROR: :1:10: expected '}'\n" + " | {'k': 'v'\n" + " | .........^", + }, + ErrorTestCase{ + .source = "Msg{?f: 1}", + .expected_error = "ERROR: :1:5: unsupported syntax '?'\n" + " | Msg{?f: 1}\n" + " | ....^", + }, + ErrorTestCase{ + .source = "Msg{1: 2}", + .expected_error = "ERROR: :1:5: expected struct field name\n" + " | Msg{1: 2}\n" + " | ....^", + }, + ErrorTestCase{ + .source = "Msg{f 10}", + .expected_error = "ERROR: :1:7: expected ':' in struct field\n" + " | Msg{f 10}\n" + " | ......^", + }, + ErrorTestCase{ + .source = "Msg{f: 10", + .expected_error = "ERROR: :1:10: expected '}'\n" + " | Msg{f: 10\n" + " | .........^", + }, + ErrorTestCase{ + .source = "f(1, 2", + .expected_error = + "ERROR: :1:7: mismatched input expecting ')'\n" + " | f(1, 2\n" + " | ......^", + }, + ErrorTestCase{ + .source = "999999999999999999999999999999999999999", + .expected_error = "ERROR: :1:1: invalid int literal\n" + " | 999999999999999999999999999999999999999\n" + " | ^", + }, + ErrorTestCase{ + .source = "999999999999999999999999999999999999999u", + .expected_error = "ERROR: :1:1: invalid uint literal\n" + " | 999999999999999999999999999999999999999u\n" + " | ^", + }, + ErrorTestCase{ + .source = "1e", + .expected_error = + "ERROR: :1:1: floating point literal missing digits after " + "exponent separator\n" + " | 1e\n" + " | ^", + }, + ErrorTestCase{ + .source = "\"unterminated", + .expected_error = "ERROR: :1:1: unterminated string literal\n" + " | \"unterminated\n" + " | ^", + }, + ErrorTestCase{ + .source = "b\"unterminated", + .expected_error = "ERROR: :1:1: unterminated bytes literal\n" + " | b\"unterminated\n" + " | ^", + }, + ErrorTestCase{ + .source = "a.?`foo`", + .expected_error = "ERROR: :1:4: unsupported syntax '`'\n" + " | a.?`foo`\n" + " | ...^", + .enable_optional_syntax = true, + .enable_quoted_identifiers = false, + }, + ErrorTestCase{ + .source = "a.`foo`()", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`foo`()\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "`foo`", + .expected_error = "ERROR: :1:1: unexpected quoted identifier\n" + " | `foo`\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "`foo`()", + .expected_error = "ERROR: :1:1: unexpected quoted identifier\n" + " | `foo`()\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.`b@c`", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`b@c`\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.``", + .expected_error = "ERROR: :1:3: unexpected quoted identifier\n" + " | a.``\n" + " | ..^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "a.`foo`", + .expected_error = "ERROR: :1:3: unsupported syntax '`'\n" + " | a.`foo`\n" + " | ..^", + .enable_quoted_identifiers = false, + }, + ErrorTestCase{ + .source = "`foo", + .expected_error = + "ERROR: :1:1: unterminated quoted identifier\n" + " | `foo\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "f(*, 1e, {2 3})", + .expected_error = + "ERROR: :1:3: unexpected token\n" + " | f(*, 1e, {2 3})\n" + " | ..^\n" + "ERROR: :1:6: floating point literal missing digits after " + "exponent separator\n" + " | f(*, 1e, {2 3})\n" + " | .....^\n" + "ERROR: :1:13: expected ':' in map entry\n" + " | f(*, 1e, {2 3})\n" + " | ............^", + }, + ErrorTestCase{ + .source = "(1 + *) + 2", + .expected_error = "ERROR: :1:6: unexpected token\n" + " | (1 + *) + 2\n" + " | .....^", + }, + ErrorTestCase{ + .source = "f(1 + *, 2)", + .expected_error = "ERROR: :1:7: unexpected token\n" + " | f(1 + *, 2)\n" + " | ......^", + }, + ErrorTestCase{ + .source = "(a. + 1)", + .expected_error = + "ERROR: :1:5: expected identifier after '.'\n" + " | (a. + 1)\n" + " | ....^", + }, + ErrorTestCase{ + .source = "f(a., 1)", + .expected_error = + "ERROR: :1:5: expected identifier after '.'\n" + " | f(a., 1)\n" + " | ....^", + }, + ErrorTestCase{ + .source = "[a., 1]", + .expected_error = + "ERROR: :1:4: expected identifier after '.'\n" + " | [a., 1]\n" + " | ...^", + }, + ErrorTestCase{ + .source = "-0x8000000000000001", + .expected_error = "ERROR: :1:2: invalid int literal\n" + " | -0x8000000000000001\n" + " | .^", + }, + ErrorTestCase{ + .source = "-0x10000000000000000", + .expected_error = "ERROR: :1:2: invalid int literal\n" + " | -0x10000000000000000\n" + " | .^", + }, + ErrorTestCase{ + .source = "-9223372036854775809", + .expected_error = "ERROR: :1:2: invalid int literal\n" + " | -9223372036854775809\n" + " | .^", + }, + ErrorTestCase{ + .source = "-999999999999999999999999999999999999999", + .expected_error = "ERROR: :1:2: invalid int literal\n" + " | -999999999999999999999999999999999999999\n" + " | .^", + }, + ErrorTestCase{ + .source = "-", + .expected_error = + "ERROR: :1:2: Syntax error: mismatched input '' " + "expecting expression\n" + " | -\n" + " | .^", + }, + ErrorTestCase{ + .source = "- *", + .expected_error = "ERROR: :1:3: unexpected token\n" + " | - *\n" + " | ..^", + }, + ErrorTestCase{ + .source = "\"😀😀😀😀😀\" ~error", + .expected_error = "ERROR: :1:9: unexpected character\n" + " | \"😀😀😀😀😀\" ~error\n" + " | ........^", + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserErrorTest, PrattParserErrorTest, + testing::ValuesIn(GetErrorTestCases())); + +TEST(PrattParserTest, SourceInfoPositionsPopulated) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, Parse("a + b")); + const cel::SourceInfo& source_info = ast->source_info(); + EXPECT_FALSE(source_info.positions().empty()); + + const cel::Expr& root = ast->root_expr(); + EXPECT_EQ(source_info.positions().at(root.id()), 2); + ASSERT_TRUE(root.has_call_expr()); + ASSERT_EQ(root.call_expr().args().size(), 2); + EXPECT_EQ(source_info.positions().at(root.call_expr().args()[0].id()), 0); + EXPECT_EQ(source_info.positions().at(root.call_expr().args()[1].id()), 4); +} + +TEST(PrattParserRecursionDepthTest, ParseRecursionDepth) { + cel::ParserOptions options; + options.max_recursion_depth = 5; + EXPECT_THAT(Parse("((((1))))", options), IsOkAndHolds(NotNull())); + EXPECT_THAT(Parse("[[[[[[1]]]]]]", options), + StatusIs(absl::StatusCode::kCancelled)); +} + +TEST(PrattParserRecursionDepthTest, SequentialScopesDoNotAccumulateDepth) { + cel::ParserOptions options; + options.max_recursion_depth = 2; + EXPECT_THAT(Parse("[1] + [2] + [3]", options), IsOkAndHolds(NotNull())); +} + +class TestParserWorker : public ParserWorker { + // Expose the protected constructor and methods for testing. + public: + using ParserWorker::GetTokenText; + using ParserWorker::ParserWorker; +}; + +TEST(ParserWorkerTest, GetTokenTextBoundsChecking) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr source, + cel::NewSource("hello world")); + cel::ParserOptions options; + TestParserWorker worker(*source, options, nullptr); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 0, .end = 5}), + "hello"); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = -1, .end = 5}), + ""); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 5, .end = 2}), + ""); + EXPECT_EQ(worker.GetTokenText( + Token{.type = TokenType::kIdent, .start = 0, .end = 100}), + ""); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc new file mode 100644 index 000000000..ff60e7962 --- /dev/null +++ b/parser/internal/pratt_parser_worker.cc @@ -0,0 +1,158 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/pratt_parser_worker.h" + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/strings/str_cat.h" +#include "common/source.h" +#include "parser/internal/lexer.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +ParserWorker::ParserWorker( + const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues) + : source_(source), + options_(options), + lexer_(source_), + parse_issues_(parse_issues) {} + +void ParserWorker::InitTokenStream() { + current_token_ = Token{.type = TokenType::kError, .start = 0, .end = 0}; + peek_token_ = NextSignificantToken(); +} + +std::string ParserWorker::GetTokenText(const Token& tok) const { + if (tok.start >= 0 && tok.end >= tok.start && + tok.end <= static_cast(source_.content().size())) { + return source_.content().ToString(tok.start, tok.end); + } + return ""; +} + +Token ParserWorker::NextSignificantToken() { + while (true) { + Token tok = lexer_.Lex(); + if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) { + continue; + } + if (tok.type == TokenType::kError) { + ReportError(tok, lexer_.GetError().message); + } + return tok; + } +} + +Token ParserWorker::NextToken() { + current_token_ = peek_token_; + if (peek_token_.type != TokenType::kEnd) { + peek_token_ = NextSignificantToken(); + } + return current_token_; +} + +bool ParserWorker::Expect(TokenType type, std::string_view msg) { + if (peek_token_.type == type) { + NextToken(); + return true; + } + if (peek_token_.type != TokenType::kError) { + std::string err_msg; + if (msg.empty()) { + std::string tok_text = GetTokenText(peek_token_); + std::string formatted_tok; + if (peek_token_.type == TokenType::kEnd) { + formatted_tok = ""; + } else { + formatted_tok = absl::StrCat("'", tok_text, "'"); + } + err_msg = absl::StrCat("mismatched input ", formatted_tok, " expecting '", + TokenTypeToString(type), "'"); + } else { + err_msg = std::string(msg); + } + ReportError(peek_token_, err_msg); + } + SynchronizeOnDelimiter(); + return false; +} + +void ParserWorker::SynchronizeOnDelimiter() { + if (is_recovery_limit_exceeded()) { + while (peek_token_.type != TokenType::kEnd) { + NextToken(); + } + return; + } + while (peek_token_.type != TokenType::kEnd) { + if (peek_token_.type == TokenType::kComma || + peek_token_.type == TokenType::kRightParen || + peek_token_.type == TokenType::kRightBracket || + peek_token_.type == TokenType::kRightBrace) { + break; + } + NextToken(); + } +} +int64_t ParserWorker::NextId(int32_t position) { + int64_t id = next_id_++; + if (position >= 0) { + positions_.insert({id, position}); + } + return id; +} + +int64_t ParserWorker::CopyId(int64_t id) { + if (id == 0) { + return 0; + } + int32_t pos = 0; + if (auto it = positions_.find(id); it != positions_.end()) { + pos = it->second; + } + return NextId(pos); +} + +void ParserWorker::EraseId(int64_t id) { + positions_.erase(id); + if (next_id_ == id + 1) { + --next_id_; + } +} + +void ParserWorker::ReportError(int32_t position, std::string_view msg) { + cel::SourceLocation loc; + if (auto found = source_.GetLocation(position); found.has_value()) { + loc = *found; + } + ReportError(loc, msg); +} + +void ParserWorker::ReportError(const SourceLocation& loc, + std::string_view msg) { + error_count_++; + if (parse_issues_ != nullptr) { + parse_issues_->push_back(cel::ParseIssue(loc, std::string(msg))); + } +} + +} // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h new file mode 100644 index 000000000..a510f32a3 --- /dev/null +++ b/parser/internal/pratt_parser_worker.h @@ -0,0 +1,860 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "common/operators.h" +#include "common/source.h" +#include "internal/lexis.h" +#include "internal/strings.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/internal/lexer.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { + +class ParserWorker { + public: + ParserWorker(const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues); + + const absl::flat_hash_map& GetNodePositions() const { + return positions_; + } + absl::Span GetLineOffsets() const { + return source_.line_offsets(); + } + bool has_errors() const { return error_count_ > 0; } + bool is_recursion_limit_exceeded() const { return recursion_limit_exceeded_; } + + protected: + const cel::Source& source() const { return source_; } + const cel::ParserOptions& options() const { return options_; } + // Token stream management + void InitTokenStream(); + Token NextSignificantToken(); + Token NextToken(); + bool Expect(TokenType type, absl::string_view msg = ""); + std::string GetTokenText(const Token& tok) const; + void SynchronizeOnDelimiter(); + + // ID and Position tracking + int64_t NextId(int32_t position); + int64_t NextId(const Token& token) { return NextId(token.start); } + int64_t NextId() { return next_id_++; } + int64_t CopyId(int64_t id); + void EraseId(int64_t id); + + // Error reporting and recovery + bool is_recovery_limit_exceeded() const { + return error_count_ >= options_.error_recovery_limit; + } + void ReportError(int32_t position, absl::string_view msg); + void ReportError(const SourceLocation& loc, absl::string_view msg); + void ReportError(const Token& token, absl::string_view msg) { + ReportError(token.start, msg); + } + + const cel::Source& source_; + cel::ParserOptions options_; + Lexer lexer_; + Token current_token_; + Token peek_token_; + int recursion_depth_ = 0; + int64_t next_id_ = 1; + absl::flat_hash_map positions_; + std::vector* absl_nullable parse_issues_; + int error_count_ = 0; + bool lexer_error_reported_ = false; + bool recursion_limit_exceeded_ = false; +}; + +// Generic Pratt parser implementation parameterized by the AST node type +// (`ExprNode`). +// +// This class implements the core recursive-descent and operator-precedence +// parsing logic for CEL without depending on a concrete expression node data +// structure. All inspection and construction of AST nodes is performed through +// `AstFactoryInterface`. +// +// See `AstFactoryInterface` documentation in `ast_factory_interface.h` for +// details on how to use this generic parser with alternative AST structures. +template +class PrattParserWorker : public ParserWorker { + public: + explicit PrattParserWorker( + const cel::Source& source, const cel::ParserOptions& options, + std::vector* absl_nullable parse_issues) + : ParserWorker(source, options, parse_issues) { + this->InitTokenStream(); + } + + ExprNode Parse(); + + private: + using CelOperator = ::google::api::expr::common::CelOperator; + + ExprNode ParseExpr(); + ExprNode ParseConditionalOr(); + ExprNode ParseConditionalAnd(); + ExprNode ParseRelation(); + ExprNode ParseCalc(int min_prec); + ExprNode ParseUnary(); + ExprNode ParseMember(); + ExprNode ParsePrimary(); + ExprNode ParseList(); + ExprNode ParseMap(); + ExprNode ParseStruct(int64_t obj_id, absl::string_view struct_name); + std::vector ParseArguments(TokenType close_token); + ExprNode ParseIntLiteral(); + ExprNode ParseUintLiteral(); + ExprNode ParseDoubleLiteral(); + ExprNode ParseStringLiteral(); + ExprNode ParseBytesLiteral(); + std::string NormalizeIdent(const Token& tok, bool allow_quoted); + std::optional ExtractStructName(const ExprNode& expr); + int32_t GetLeftmostPosition(const ExprNode& expr); + ExprNode BalancedTree(absl::string_view op, std::vector& terms, + const std::vector& ops, int lo, int hi); + ExprNode BalanceLogical(absl::string_view op, std::vector terms, + std::vector ops, bool enable_variadic); + + AstFactoryInterface ast_factory_; +}; + +template +ExprNode PrattParserWorker::Parse() { + ExprNode expr = ParseExpr(); + if (is_recursion_limit_exceeded()) { + return expr; + } + if (peek_token_.type != TokenType::kEnd && + peek_token_.type != TokenType::kError) { + ReportError(peek_token_, "unexpected token after expression"); + } + return expr; +} + +template +ExprNode PrattParserWorker::ParseExpr() { + if (recursion_limit_exceeded_) { + return ExprNode(); + } + if (recursion_depth_ >= options_.max_recursion_depth) { + recursion_limit_exceeded_ = true; + return ExprNode(); + } + recursion_depth_++; + ExprNode lhs = ParseConditionalOr(); + if (peek_token_.type == TokenType::kQuestion) { + NextToken(); + int64_t op_id = NextId(); + ExprNode true_expr = ParseConditionalOr(); + if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { + recursion_depth_--; + return lhs; + } + ExprNode false_expr = ParseExpr(); + recursion_depth_--; + return ast_factory_.NewCall( + op_id, CelOperator::CONDITIONAL, + std::vector{std::move(lhs), std::move(true_expr), + std::move(false_expr)}); + } + recursion_depth_--; + return lhs; +} + +// Parses logical OR expressions (e.g., `a || b || c`). +template +ExprNode PrattParserWorker::ParseConditionalOr() { + ExprNode lhs = ParseConditionalAnd(); + if (peek_token_.type != TokenType::kLogicalOr) { + return lhs; + } + std::vector terms; + std::vector ops; + terms.push_back(std::move(lhs)); + while (peek_token_.type == TokenType::kLogicalOr) { + Token op_tok = NextToken(); + ExprNode rhs = ParseConditionalAnd(); + ops.push_back(NextId(op_tok)); + terms.push_back(std::move(rhs)); + } + return BalanceLogical(CelOperator::LOGICAL_OR, std::move(terms), + std::move(ops), + options_.enable_variadic_logical_operators); +} + +// Parses logical AND expressions (e.g., `a && b && c`). +template +ExprNode PrattParserWorker::ParseConditionalAnd() { + ExprNode lhs = ParseRelation(); + if (peek_token_.type != TokenType::kLogicalAnd) { + return lhs; + } + std::vector terms; + std::vector ops; + terms.push_back(std::move(lhs)); + while (peek_token_.type == TokenType::kLogicalAnd) { + Token op_tok = NextToken(); + ExprNode rhs = ParseRelation(); + ops.push_back(NextId(op_tok)); + terms.push_back(std::move(rhs)); + } + return BalanceLogical(CelOperator::LOGICAL_AND, std::move(terms), + std::move(ops), + options_.enable_variadic_logical_operators); +} + +// Parses relational & equality ops (e.g., `a < b`, `x == y`, `a in b`). +template +ExprNode PrattParserWorker::ParseRelation() { + ExprNode lhs = ParseCalc(0); + while (true) { + TokenType tok = peek_token_.type; + absl::string_view op_name; + switch (tok) { + case TokenType::kLess: + op_name = CelOperator::LESS; + break; + case TokenType::kLessEqual: + op_name = CelOperator::LESS_EQUALS; + break; + case TokenType::kGreater: + op_name = CelOperator::GREATER; + break; + case TokenType::kGreaterEqual: + op_name = CelOperator::GREATER_EQUALS; + break; + case TokenType::kEqualEqual: + op_name = CelOperator::EQUALS; + break; + case TokenType::kExclamationEqual: + op_name = CelOperator::NOT_EQUALS; + break; + case TokenType::kIn: + op_name = CelOperator::IN; + break; + default: + return lhs; + } + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode rhs = ParseCalc(0); + lhs = ast_factory_.NewCall( + op_id, std::string(op_name), + std::vector{std::move(lhs), std::move(rhs)}); + } + return lhs; +} + +// Parses arithmetic calculation expressions (e.g., `a + b * c`, `x - y % z`). +template +ExprNode PrattParserWorker::ParseCalc(int min_prec) { + ExprNode lhs = ParseUnary(); + while (true) { + TokenType tok = peek_token_.type; + int prec = 0; + absl::string_view op_name; + if (tok == TokenType::kAsterisk) { + prec = 2; + op_name = CelOperator::MULTIPLY; + } else if (tok == TokenType::kSlash) { + prec = 2; + op_name = CelOperator::DIVIDE; + } else if (tok == TokenType::kPercent) { + prec = 2; + op_name = CelOperator::MODULO; + } else if (tok == TokenType::kPlus) { + prec = 1; + op_name = CelOperator::ADD; + } else if (tok == TokenType::kMinus) { + prec = 1; + op_name = CelOperator::SUBTRACT; + } else { + break; + } + + if (prec < min_prec) break; + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode rhs = ParseCalc(prec + 1); + lhs = ast_factory_.NewCall( + op_id, std::string(op_name), + std::vector{std::move(lhs), std::move(rhs)}); + } + return lhs; +} + +// Parses unary logical NOT and negation expressions (e.g., `!a`, `-b`). +template +ExprNode PrattParserWorker::ParseUnary() { + TokenType tok = peek_token_.type; + if (tok == TokenType::kExclamation) { + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode operand = ParseUnary(); + return ast_factory_.NewCall(op_id, std::string(CelOperator::LOGICAL_NOT), + std::vector{std::move(operand)}); + } + if (tok == TokenType::kMinus) { + Token op = NextToken(); + if (peek_token_.type == TokenType::kInt) { + Token lit_tok = NextToken(); + std::string text = GetTokenText(lit_tok); + int64_t int_val = 0; + bool success = false; + if (absl::StartsWith(text, "0x") || absl::StartsWith(text, "0X")) { + uint64_t uint_val = 0; + if (absl::SimpleHexAtoi(text, &uint_val)) { + if (uint_val <= uint64_t{0x8000000000000000}) { + if (uint_val == uint64_t{0x8000000000000000}) { + int_val = std::numeric_limits::min(); + } else { + int_val = -static_cast(uint_val); + } + success = true; + } + } + } else { + std::string val = absl::StrCat("-", text); + success = absl::SimpleAtoi(val, &int_val); + } + if (success) { + return ast_factory_.NewIntConst(NextId(op.start), int_val); + } + ReportError(lit_tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); + } + if (peek_token_.type == TokenType::kFloat) { + Token lit_tok = NextToken(); + std::string val = absl::StrCat("-", GetTokenText(lit_tok)); + double double_val = 0.0; + if (absl::SimpleAtod(val, &double_val)) { + return ast_factory_.NewDoubleConst(NextId(op.start), double_val); + } + ReportError(lit_tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); + } + // Regular negate call + int64_t op_id = NextId(op); + ExprNode operand = ParseUnary(); + return ast_factory_.NewCall(op_id, std::string(CelOperator::NEGATE), + std::vector{std::move(operand)}); + } + return ParseMember(); +} + +// Parses member calls & indexing (e.g., `a.b`, `a.b(c)`, `a[b]`, `a.?b`). +template +ExprNode PrattParserWorker::ParseMember() { + ExprNode lhs = ParsePrimary(); + while (true) { + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot) { + Token dot_tok = NextToken(); + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(dot_tok, "unsupported syntax '.?'"); + } + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + if (id_tok.type != TokenType::kError) { + ReportError(id_tok, "expected identifier after '.'"); + } + SynchronizeOnDelimiter(); + return lhs; + } + bool is_member_call = peek_token_.type == TokenType::kLeftParen; + std::string id_text = + NormalizeIdent(id_tok, /*allow_quoted=*/!is_member_call); + if (optional) { + int64_t op_id = NextId(dot_tok); + lhs = ast_factory_.NewCall( + op_id, "_?._", + std::vector{ + std::move(lhs), + ast_factory_.NewStringConst(NextId(id_tok), id_text)}); + } else if (peek_token_.type == TokenType::kLeftParen) { + Token lparen = NextToken(); + int64_t call_id = NextId(lparen); + std::vector args = ParseArguments(TokenType::kRightParen); + lhs = ast_factory_.NewMemberCall(call_id, id_text, std::move(lhs), + std::move(args)); + } else { + lhs = ast_factory_.NewSelect(NextId(dot_tok), std::move(lhs), id_text); + } + } else if (tok == TokenType::kLeftBracket) { + Token bracket_tok = NextToken(); + int64_t op_id = NextId(bracket_tok); + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(bracket_tok, "unsupported syntax '?'"); + } + } + ExprNode index = ParseExpr(); + Expect(TokenType::kRightBracket, "expected ']'"); + lhs = ast_factory_.NewCall( + op_id, optional ? "_[?_]" : CelOperator::INDEX, + std::vector{std::move(lhs), std::move(index)}); + } else if (tok == TokenType::kLeftBrace) { + int32_t struct_pos = GetLeftmostPosition(lhs); + if (auto struct_name = ExtractStructName(lhs); struct_name.has_value()) { + lhs = ParseStruct(NextId(struct_pos), *struct_name); + } else { + break; + } + } else { + break; + } + } + return lhs; +} + +// Parses primary expressions & literals (e.g., `null`, `true`, `x`, +// `has(x.y)`). +template +ExprNode PrattParserWorker::ParsePrimary() { + ExprNode expr; + TokenType tok_type = peek_token_.type; + if (tok_type == TokenType::kLeftParen) { + NextToken(); + expr = ParseExpr(); + Expect(TokenType::kRightParen); + } else if (tok_type == TokenType::kNull) { + Token tok = NextToken(); + expr = ast_factory_.NewNullConst(NextId(tok)); + } else if (tok_type == TokenType::kTrue || tok_type == TokenType::kFalse) { + Token tok = NextToken(); + expr = ast_factory_.NewBoolConst(NextId(tok), tok_type == TokenType::kTrue); + } else if (tok_type == TokenType::kInt) { + expr = ParseIntLiteral(); + } else if (tok_type == TokenType::kUint) { + expr = ParseUintLiteral(); + } else if (tok_type == TokenType::kFloat) { + expr = ParseDoubleLiteral(); + } else if (tok_type == TokenType::kString) { + expr = ParseStringLiteral(); + } else if (tok_type == TokenType::kBytes) { + expr = ParseBytesLiteral(); + } else if (tok_type == TokenType::kLeftBracket) { + expr = ParseList(); + } else if (tok_type == TokenType::kLeftBrace) { + expr = ParseMap(); + } else if (tok_type == TokenType::kDot || tok_type == TokenType::kIdent || + tok_type == TokenType::kReservedWord) { + bool leading_dot = false; + Token first_tok = peek_token_; + if (tok_type == TokenType::kDot) { + NextToken(); + leading_dot = true; + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + if (id_tok.type != TokenType::kError) { + ReportError(id_tok, "expected identifier"); + } + expr = ast_factory_.NewUnspecified(NextId(id_tok)); + } else { + std::string id_text = NormalizeIdent(id_tok, /*allow_quoted=*/false); + if (cel::internal::LexisIsReserved(id_text)) { + ReportError(id_tok, + absl::StrFormat("reserved identifier: %s", id_text)); + } + std::string name = + leading_dot ? absl::StrCat(".", id_text) : std::string(id_text); + int64_t id = NextId(leading_dot ? first_tok : id_tok); + if (peek_token_.type == TokenType::kLeftParen) { + NextToken(); + std::vector args = ParseArguments(TokenType::kRightParen); + expr = ast_factory_.NewCall(id, name, std::move(args)); + } else { + expr = ast_factory_.NewIdent(id, std::move(name)); + } + } + } else { + Token bad_tok = NextToken(); + if (bad_tok.type != TokenType::kError) { + if (bad_tok.type == TokenType::kEnd) { + ReportError( + bad_tok, + "Syntax error: mismatched input '' expecting expression"); + } else { + ReportError(bad_tok, "unexpected token"); + } + } + expr = ast_factory_.NewUnspecified(NextId(bad_tok)); + } + + return expr; +} + +// Parses list creation literals (e.g., `[1, 2, ?3]`). +template +ExprNode PrattParserWorker::ParseList() { + Token open_tok = NextToken(); + int64_t list_id = NextId(open_tok); + ListNodeBuilder builder = ast_factory_.NewListBuilder(list_id); + while (peek_token_.type != TokenType::kRightBracket && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + } + ExprNode elem = ParseExpr(); + builder.Add(std::move(elem), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBracket, "expected ']'"); + return builder.Build(); +} + +// Parses map creation literals (e.g., `{"key": "value", ?"opt_key": 42}`). +template +ExprNode PrattParserWorker::ParseMap() { + Token open_tok = NextToken(); + int64_t map_id = NextId(open_tok); + MapNodeBuilder builder = ast_factory_.NewMapBuilder(map_id); + while (peek_token_.type != TokenType::kRightBrace && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + Token key_start = peek_token_; + if (key_start.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + key_start = peek_token_; + } + ExprNode key = ParseExpr(); + Token colon = peek_token_; + if (!Expect(TokenType::kColon, "expected ':' in map entry")) { + break; + } + int64_t entry_id = NextId(colon); + ExprNode val = ParseExpr(); + builder.Add(entry_id, std::move(key), std::move(val), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBrace, "expected '}'"); + return builder.Build(); +} + +// Parses struct literas for a type name (e.g., `Msg{field: 10, ?opt: "val"}`). +template +ExprNode PrattParserWorker::ParseStruct( + int64_t obj_id, absl::string_view struct_name) { + Token open_tok = NextToken(); + StructNodeBuilder builder = + ast_factory_.NewStructBuilder(obj_id, std::string(struct_name)); + while (peek_token_.type != TokenType::kRightBrace && + peek_token_.type != TokenType::kEnd) { + bool optional = false; + if (peek_token_.type == TokenType::kQuestion) { + Token q = NextToken(); + optional = true; + if (!options_.enable_optional_syntax) { + ReportError(q, "unsupported syntax '?'"); + } + } + Token field_tok = NextToken(); + if (field_tok.type != TokenType::kIdent && + field_tok.type != TokenType::kReservedWord) { + ReportError(field_tok, "expected struct field name"); + SynchronizeOnDelimiter(); + break; + } + std::string field_name = NormalizeIdent(field_tok, /*allow_quoted=*/true); + Token colon = peek_token_; + if (!Expect(TokenType::kColon, "expected ':' in struct field")) { + break; + } + int64_t field_id = NextId(colon); + ExprNode val = ParseExpr(); + builder.Add(field_id, std::move(field_name), std::move(val), optional); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + } else { + break; + } + } + Expect(TokenType::kRightBrace, "expected '}'"); + return builder.Build(); +} + +// Parses comma-separated arguments (e.g., `(arg1, arg2)` in call). +template +std::vector PrattParserWorker::ParseArguments( + TokenType close_token) { + std::vector args; + if (peek_token_.type != close_token && peek_token_.type != TokenType::kEnd) { + while (true) { + args.push_back(ParseExpr()); + if (peek_token_.type == TokenType::kComma) { + NextToken(); + if (peek_token_.type == close_token) { + break; + } + continue; + } + break; + } + } + Expect(close_token); + return args; +} + +// Parses decimal & hexadecimal ints (e.g., `42`, `0x1A`). +template +ExprNode PrattParserWorker::ParseIntLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + int64_t int_val = 0; + if (absl::StartsWith(value, "0x") || absl::StartsWith(value, "0X")) { + if (absl::SimpleHexAtoi(value, &int_val)) { + return ast_factory_.NewIntConst(NextId(tok), int_val); + } + } else if (absl::SimpleAtoi(value, &int_val)) { + return ast_factory_.NewIntConst(NextId(tok), int_val); + } + ReportError(tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses unsigned ints (e.g., `42u`, `0x1Au`). +template +ExprNode PrattParserWorker::ParseUintLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + if (!value.empty() && (value.back() == 'u' || value.back() == 'U')) { + value.pop_back(); + } + uint64_t uint_val = 0; + if (absl::StartsWith(value, "0x") || absl::StartsWith(value, "0X")) { + if (absl::SimpleHexAtoi(value, &uint_val)) { + return ast_factory_.NewUintConst(NextId(tok), uint_val); + } + } else if (absl::SimpleAtoi(value, &uint_val)) { + return ast_factory_.NewUintConst(NextId(tok), uint_val); + } + ReportError(tok, "invalid uint literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses floating-point numbers (e.g., `3.14159`, `1e-10`). +template +ExprNode PrattParserWorker::ParseDoubleLiteral() { + Token tok = NextToken(); + std::string value(GetTokenText(tok)); + double double_val = 0.0; + if (absl::SimpleAtod(value, &double_val)) { + return ast_factory_.NewDoubleConst(NextId(tok), double_val); + } + ReportError(tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(tok)); +} + +// Parses string literals (e.g., `"hello"`, `'world'`, `"""multi"""`). +template +ExprNode PrattParserWorker::ParseStringLiteral() { + Token tok = NextToken(); + absl::StatusOr status_or_val = + cel::internal::ParseStringLiteral(GetTokenText(tok)); + if (!status_or_val.ok()) { + ReportError(tok, status_or_val.status().message()); + return ast_factory_.NewUnspecified(NextId(tok)); + } + return ast_factory_.NewStringConst(NextId(tok), std::move(*status_or_val)); +} + +// Parses byte sequence literals (e.g., `b"hello"`, `b'world'`). +template +ExprNode PrattParserWorker::ParseBytesLiteral() { + Token tok = NextToken(); + absl::StatusOr status_or_val = + cel::internal::ParseBytesLiteral(GetTokenText(tok)); + if (!status_or_val.ok()) { + ReportError(tok, status_or_val.status().message()); + return ast_factory_.NewUnspecified(NextId(tok)); + } + return ast_factory_.NewBytesConst(NextId(tok), std::move(*status_or_val)); +} + +// Normalizes regular & quoted identifiers (e.g., `foo`, `` `quoted.ident` ``). +template +std::string PrattParserWorker::NormalizeIdent(const Token& tok, + bool allow_quoted) { + std::string text = GetTokenText(tok); + if (text.empty()) return ""; + if (text.front() == '`') { + if (!allow_quoted) { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + if (!options_.enable_quoted_identifiers) { + ReportError(tok, "unsupported syntax '`'"); + } + if (text.size() < 2 || text.back() != '`') { + ReportError(tok, "unterminated quoted identifier"); + return ""; + } + // Validate the quoted identifier syntax: + // ESC_IDENTIFIER : '`' (LETTER | DIGIT | '_' | '.' | '-' | '/' | ' ')+ '`'; + std::string_view inner(text.data() + 1, text.size() - 2); + if (inner.empty()) { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + for (char c : inner) { + if (!absl::ascii_isalnum(static_cast(c)) && c != '_' && + c != '.' && c != '-' && c != '/' && c != ' ') { + ReportError(tok, "unexpected quoted identifier"); + return ""; + } + } + return std::string(inner); + } + return std::string(text); +} + +// Extracts qualified type names (e.g., `a.b.Msg` from `a.b.Msg{}`). +template +std::optional PrattParserWorker::ExtractStructName( + const ExprNode& expr) { + if (ast_factory_.IsConst(expr)) { + return std::nullopt; + } + if (ast_factory_.IsIdent(expr)) { + std::string name(ast_factory_.GetIdentName(expr)); + EraseId(ast_factory_.GetId(expr)); + return name; + } + if (ast_factory_.IsSelect(expr)) { + if (ast_factory_.IsPresenceTest(expr)) return std::nullopt; + const ExprNode* operand = ast_factory_.GetSelectOperand(expr); + if (operand == nullptr) return std::nullopt; + EraseId(ast_factory_.GetId(expr)); + absl::optional prefix = ExtractStructName(*operand); + if (!prefix) return std::nullopt; + std::string name = + absl::StrCat(*prefix, ".", ast_factory_.GetSelectField(expr)); + return name; + } + return std::nullopt; +} + +template +int32_t PrattParserWorker::GetLeftmostPosition(const ExprNode& expr) { + if (ast_factory_.IsIdent(expr)) { + auto it = positions_.find(ast_factory_.GetId(expr)); + return it != positions_.end() ? it->second : 0; + } + if (ast_factory_.IsSelect(expr)) { + return GetLeftmostPosition(*ast_factory_.GetSelectOperand(expr)); + } + auto it = positions_.find(ast_factory_.GetId(expr)); + return it != positions_.end() ? it->second : 0; +} + +// Recursively constructs a balanced binary AST tree for chained associative +// operators (e.g., chains of `+` or `*`). To prevent deep recursion and +// stack overflow during evaluation of expressions like `a + b + c + d`, this +// method splits the operand terms in half at midpoint `(lo + hi + 1) / 2` and +// combines the left and right subtrees with binary call nodes. +template +ExprNode PrattParserWorker::BalancedTree( + absl::string_view op, std::vector& terms, + const std::vector& ops, int lo, int hi) { + int mid = (lo + hi + 1) / 2; + std::vector arguments; + arguments.reserve(2); + if (mid == lo) { + arguments.push_back(std::move(terms[mid])); + } else { + arguments.push_back(BalancedTree(op, terms, ops, lo, mid - 1)); + } + if (mid == hi) { + arguments.push_back(std::move(terms[mid + 1])); + } else { + arguments.push_back(BalancedTree(op, terms, ops, mid + 1, hi)); + } + return ast_factory_.NewCall(ops[mid], std::string(op), std::move(arguments)); +} + +// Constructs the AST representation for chained logical operators (e.g., +// `a && b && c` or `a || b || c`). When variadic logical ops are enabled +// (`enable_variadic = true`), it creates a single N-ary function call node +// containing all terms as arguments (e.g., `_&&_(a, b, c)`). Otherwise, it +// delegates to `BalancedTree` to produce a balanced binary tree of logical +// operations `(a && b) && c`. +template +ExprNode PrattParserWorker::BalanceLogical( + absl::string_view op, std::vector terms, std::vector ops, + bool enable_variadic) { + if (terms.size() == 1) { + return std::move(terms[0]); + } + if (enable_variadic) { + return ast_factory_.NewCall(ops[0], std::string(op), std::move(terms)); + } + return BalancedTree(op, terms, ops, 0, ops.size() - 1); +} + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ From 68aded818e66c20c888db3b8a9314ad9441c9159 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 23 Jul 2026 17:37:20 -0700 Subject: [PATCH 133/350] [Pratt Parser] Add benchmark to track optimizaitons PiperOrigin-RevId: 953047231 --- parser/internal/BUILD | 17 + parser/internal/pratt_parser_benchmark.cc | 445 ++++++++++++++++++++++ 2 files changed, 462 insertions(+) create mode 100644 parser/internal/pratt_parser_benchmark.cc diff --git a/parser/internal/BUILD b/parser/internal/BUILD index 0d2af99d2..d5e201bf3 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -160,3 +160,20 @@ cc_test( "@com_google_absl//absl/strings:string_view", ], ) + +cc_test( + name = "pratt_parser_benchmark", + srcs = ["pratt_parser_benchmark.cc"], + tags = ["benchmark"], + deps = [ + ":pratt_parser", + "//common:source", + "//internal:benchmark", + "//internal:testing", + "//parser", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + ], +) diff --git a/parser/internal/pratt_parser_benchmark.cc b/parser/internal/pratt_parser_benchmark.cc new file mode 100644 index 000000000..952585306 --- /dev/null +++ b/parser/internal/pratt_parser_benchmark.cc @@ -0,0 +1,445 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include "common/source.h" +#include "internal/benchmark.h" +#include "parser/internal/pratt_parser.h" +#include "parser/options.h" +#include "parser/parser.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { +namespace { + +enum class ParserImplType { + kPratt, + kAntlr, +}; + +std::unique_ptr CreateParser(ParserImplType type, + cel::ParserOptions options) { + options.max_recursion_depth = 512; + std::unique_ptr builder; + if (type == ParserImplType::kPratt) { + builder = NewPrattParserBuilder(options); + } else { + builder = cel::NewParserBuilder(options); + } + auto parser = builder->Build(); + ABSL_DCHECK_OK(parser.status()); + return std::move(*parser); +} + +// ----------------------------------------------------------------------------- +// Workload 1: Common Representative CEL Expressions +// ----------------------------------------------------------------------------- +void BM_ParseCommon(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + options.enable_optional_syntax = true; + options.enable_quoted_identifiers = true; + auto parser = CreateParser(type, options); + + const std::vector test_cases = { + "x * 2 + y / 3", + "foo.bar.baz(1, 2, \"abc\")", + "a > 5 && b < 10 || c == \"xyz\"", + "x ? y : z", + "{\"foo\": 1, \"bar\": [2, 3]}", + "has(foo.bar) && [1, 2, 3].all(x, x > 0)", + }; + + for (auto _ : state) { + for (const auto& expr : test_cases) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } + } +} + +void BM_Pratt_ParseCommon(benchmark::State& state) { + BM_ParseCommon(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseCommon(benchmark::State& state) { + BM_ParseCommon(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseCommon); +BENCHMARK(BM_Antlr_ParseCommon); + +// ----------------------------------------------------------------------------- +// Workload 2: Deep Left-Associative Arithmetic Chains ("a + a + a...") +// ----------------------------------------------------------------------------- +std::string BuildArithmeticChain(int length) { + std::string expr = "a"; + for (int i = 1; i < length; ++i) { + absl::StrAppend(&expr, " + a"); + } + return expr; +} + +void BM_ParseArithmeticChain(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildArithmeticChain(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseArithmeticChain(benchmark::State& state) { + BM_ParseArithmeticChain(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseArithmeticChain(benchmark::State& state) { + BM_ParseArithmeticChain(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseArithmeticChain)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseArithmeticChain)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 3: Wide Logical Expression Chains ("a || a || a...") +// ----------------------------------------------------------------------------- +std::string BuildLogicalChain(int length) { + std::string expr = "a"; + for (int i = 1; i < length; ++i) { + absl::StrAppend(&expr, " || a"); + } + return expr; +} + +void BM_ParseLogicalChain(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildLogicalChain(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseLogicalChain(benchmark::State& state) { + BM_ParseLogicalChain(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseLogicalChain(benchmark::State& state) { + BM_ParseLogicalChain(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseLogicalChain)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseLogicalChain)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 4: Deep Member Access Chains ("a.f.f.f...") +// ----------------------------------------------------------------------------- +std::string BuildMemberChain(int length) { + std::string expr = "a"; + for (int i = 1; i < length; ++i) { + absl::StrAppend(&expr, ".f"); + } + return expr; +} + +void BM_ParseMemberChain(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildMemberChain(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseMemberChain(benchmark::State& state) { + BM_ParseMemberChain(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseMemberChain(benchmark::State& state) { + BM_ParseMemberChain(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseMemberChain)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseMemberChain)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 5: Deeply Nested Parentheses ("(((((a)))))") +// ----------------------------------------------------------------------------- +std::string BuildNestedParentheses(int depth) { + std::string prefix(depth, '('); + std::string suffix(depth, ')'); + return absl::StrCat(prefix, "a", suffix); +} + +void BM_ParseNestedParentheses(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildNestedParentheses(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseNestedParentheses(benchmark::State& state) { + BM_ParseNestedParentheses(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseNestedParentheses(benchmark::State& state) { + BM_ParseNestedParentheses(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseNestedParentheses)->Arg(10)->Arg(50); +BENCHMARK(BM_Antlr_ParseNestedParentheses)->Arg(10)->Arg(50); + +// ----------------------------------------------------------------------------- +// Workload 6: Common Representative Expressions with Syntax Errors +// ----------------------------------------------------------------------------- +void BM_ParseCommonSyntaxErrors(benchmark::State& state, ParserImplType type) { + cel::ParserOptions options; + options.enable_optional_syntax = true; + options.enable_quoted_identifiers = true; + auto parser = CreateParser(type, options); + + const std::vector test_cases = { + "x * 2 + y /", + "foo.bar.baz(1, 2, \"abc\"", + "a > 5 && && b < 10 || c == \"xyz\"", + "x ? y :", + "{\"foo\": 1, \"bar\": [2, 3", + "has(foo.bar) && [1, 2, 3].all(x, +)", + "a. + b", + "Msg{f 10}", + "(1 + 2 * (3 + )", + }; + + for (auto _ : state) { + for (const auto& expr : test_cases) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } + } +} + +void BM_Pratt_ParseCommonSyntaxErrors(benchmark::State& state) { + BM_ParseCommonSyntaxErrors(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseCommonSyntaxErrors(benchmark::State& state) { + BM_ParseCommonSyntaxErrors(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseCommonSyntaxErrors); +BENCHMARK(BM_Antlr_ParseCommonSyntaxErrors); + +// ----------------------------------------------------------------------------- +// Workload 7: Deep Left-Associative Arithmetic Chains with Syntax Error +// ("a + a + ... + ") +// ----------------------------------------------------------------------------- +std::string BuildArithmeticChainSyntaxError(int length) { + std::string expr = BuildArithmeticChain(length); + absl::StrAppend(&expr, " +"); + return expr; +} + +void BM_ParseArithmeticChainSyntaxError(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildArithmeticChainSyntaxError(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseArithmeticChainSyntaxError(benchmark::State& state) { + BM_ParseArithmeticChainSyntaxError(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseArithmeticChainSyntaxError(benchmark::State& state) { + BM_ParseArithmeticChainSyntaxError(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseArithmeticChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseArithmeticChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 8: Wide Logical Expression Chains with Syntax Error +// ("a || a || ... || ") +// ----------------------------------------------------------------------------- +std::string BuildLogicalChainSyntaxError(int length) { + std::string expr = BuildLogicalChain(length); + absl::StrAppend(&expr, " ||"); + return expr; +} + +void BM_ParseLogicalChainSyntaxError(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildLogicalChainSyntaxError(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseLogicalChainSyntaxError(benchmark::State& state) { + BM_ParseLogicalChainSyntaxError(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseLogicalChainSyntaxError(benchmark::State& state) { + BM_ParseLogicalChainSyntaxError(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseLogicalChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseLogicalChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 9: Deep Member Access Chains with Syntax Error ("a.f.f...f.") +// ----------------------------------------------------------------------------- +std::string BuildMemberChainSyntaxError(int length) { + std::string expr = BuildMemberChain(length); + absl::StrAppend(&expr, "."); + return expr; +} + +void BM_ParseMemberChainSyntaxError(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildMemberChainSyntaxError(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseMemberChainSyntaxError(benchmark::State& state) { + BM_ParseMemberChainSyntaxError(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseMemberChainSyntaxError(benchmark::State& state) { + BM_ParseMemberChainSyntaxError(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseMemberChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseMemberChainSyntaxError)->Arg(10)->Arg(50)->Arg(100); + +// ----------------------------------------------------------------------------- +// Workload 10: Deeply Nested Parentheses with Syntax Error ("(((((a") +// ----------------------------------------------------------------------------- +std::string BuildNestedParenthesesSyntaxError(int depth) { + std::string prefix(depth, '('); + return absl::StrCat(prefix, "a"); +} + +void BM_ParseNestedParenthesesSyntaxError(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildNestedParenthesesSyntaxError(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseNestedParenthesesSyntaxError(benchmark::State& state) { + BM_ParseNestedParenthesesSyntaxError(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseNestedParenthesesSyntaxError(benchmark::State& state) { + BM_ParseNestedParenthesesSyntaxError(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseNestedParenthesesSyntaxError)->Arg(10)->Arg(50); +BENCHMARK(BM_Antlr_ParseNestedParenthesesSyntaxError)->Arg(10)->Arg(50); + +// ----------------------------------------------------------------------------- +// Workload 11: Repeated Syntax Errors ("f(*, *, *, ...)") +// ----------------------------------------------------------------------------- +std::string BuildRepeatedSyntaxErrors(int length) { + std::string expr = "f(*"; + for (int i = 1; i < length; ++i) { + absl::StrAppend(&expr, ", *"); + } + absl::StrAppend(&expr, ")"); + return expr; +} + +void BM_ParseRepeatedSyntaxErrors(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildRepeatedSyntaxErrors(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK(!ast.ok()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseRepeatedSyntaxErrors(benchmark::State& state) { + BM_ParseRepeatedSyntaxErrors(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseRepeatedSyntaxErrors(benchmark::State& state) { + BM_ParseRepeatedSyntaxErrors(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseRepeatedSyntaxErrors)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK(BM_Antlr_ParseRepeatedSyntaxErrors)->Arg(10)->Arg(50)->Arg(100); + +} // namespace +} // namespace cel::parser_internal From 5c9cda76521f6124c6cc3eb208315befecdfedbd Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 23 Jul 2026 18:05:47 -0700 Subject: [PATCH 134/350] [Pratt Parser] Add deep copy/replace functionality to AstFactoryInterface PiperOrigin-RevId: 953058710 --- parser/internal/BUILD | 13 +- parser/internal/ast_factory.cc | 269 ++++++++++++++++++++++++ parser/internal/ast_factory.h | 112 +++------- parser/internal/ast_factory_interface.h | 37 ++-- parser/internal/ast_factory_test.cc | 199 +++++++++++++++++- 5 files changed, 534 insertions(+), 96 deletions(-) create mode 100644 parser/internal/ast_factory.cc diff --git a/parser/internal/BUILD b/parser/internal/BUILD index d5e201bf3..535a1bb79 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -23,16 +23,24 @@ licenses(["notice"]) cc_library( name = "ast_factory_interface", hdrs = ["ast_factory_interface.h"], + deps = [ + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/status:statusor", + ], ) cc_library( name = "ast_factory", + srcs = ["ast_factory.cc"], hdrs = ["ast_factory.h"], deps = [ ":ast_factory_interface", - "//common:constant", "//common:expr", "//common:expr_factory", + "//internal:status_macros", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", ], ) @@ -119,8 +127,11 @@ cc_test( srcs = ["ast_factory_test.cc"], deps = [ ":ast_factory", + "//common:constant", "//common:expr", "//internal:testing", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings:string_view", ], ) diff --git a/parser/internal/ast_factory.cc b/parser/internal/ast_factory.cc new file mode 100644 index 000000000..b339f06b4 --- /dev/null +++ b/parser/internal/ast_factory.cc @@ -0,0 +1,269 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/ast_factory.h" + +#include +#include +#include +#include + +#include "absl/functional/function_ref.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/expr.h" +#include "internal/status_macros.h" + +namespace cel::parser_internal { + +ListNodeBuilder::ListNodeBuilder(int64_t id) { + expr_.set_id(id); + expr_.mutable_list_expr(); +} + +ListNodeBuilder& ListNodeBuilder::Add(cel::Expr element, + bool optional) { + cel::ListExpr& list_val = expr_.mutable_list_expr(); + cel::ListExprElement expr_element; + expr_element.set_expr(std::move(element)); + expr_element.set_optional(optional); + list_val.mutable_elements().push_back(std::move(expr_element)); + return *this; +} + +cel::Expr ListNodeBuilder::Build() { return std::move(expr_); } + +MapNodeBuilder::MapNodeBuilder(int64_t id) { + expr_.set_id(id); + expr_.mutable_map_expr(); +} + +MapNodeBuilder& MapNodeBuilder::Add(int64_t id, + cel::Expr key, + cel::Expr value, + bool optional) { + cel::MapExpr& map_val = expr_.mutable_map_expr(); + cel::MapExprEntry entry; + entry.set_id(id); + entry.set_key(std::move(key)); + entry.set_value(std::move(value)); + entry.set_optional(optional); + map_val.mutable_entries().push_back(std::move(entry)); + return *this; +} + +cel::Expr MapNodeBuilder::Build() { return std::move(expr_); } + +StructNodeBuilder::StructNodeBuilder(int64_t id, std::string name) { + expr_.set_id(id); + expr_.mutable_struct_expr().set_name(std::move(name)); +} + +StructNodeBuilder& StructNodeBuilder::Add( + int64_t id, std::string name, cel::Expr value, bool optional) { + cel::StructExpr& struct_val = expr_.mutable_struct_expr(); + cel::StructExprField field; + field.set_id(id); + field.set_name(std::move(name)); + field.set_value(std::move(value)); + field.set_optional(optional); + struct_val.mutable_fields().push_back(std::move(field)); + return *this; +} + +cel::Expr StructNodeBuilder::Build() { return std::move(expr_); } + +int64_t AstFactoryInterface::GetId(const cel::Expr& expr) const { + return expr.id(); +} + +bool AstFactoryInterface::IsEmpty(const cel::Expr& expr) const { + return expr.id() == 0; +} + +bool AstFactoryInterface::IsConst(const cel::Expr& expr) const { + return expr.has_const_expr(); +} + +bool AstFactoryInterface::IsIdent(const cel::Expr& expr) const { + return expr.has_ident_expr(); +} + +absl::string_view AstFactoryInterface::GetIdentName( + const cel::Expr& expr) const { + return expr.has_ident_expr() ? absl::string_view(expr.ident_expr().name()) + : absl::string_view(); +} + +bool AstFactoryInterface::IsSelect(const cel::Expr& expr) const { + return expr.has_select_expr(); +} + +bool AstFactoryInterface::IsPresenceTest( + const cel::Expr& expr) const { + return expr.has_select_expr() && expr.select_expr().test_only(); +} + +const cel::Expr* AstFactoryInterface::GetSelectOperand( + const cel::Expr& expr) const { + return expr.has_select_expr() ? &expr.select_expr().operand() : nullptr; +} + +absl::string_view AstFactoryInterface::GetSelectField( + const cel::Expr& expr) const { + return expr.has_select_expr() ? absl::string_view(expr.select_expr().field()) + : absl::string_view(); +} + +absl::StatusOr AstFactoryInterface::CopyAndReplace( + const cel::Expr& expr, + absl::FunctionRef(const cel::Expr&)> replacer, + int max_recursion_depth) const { + if (max_recursion_depth <= 0) { + return absl::InvalidArgumentError("recursion limit exceeded"); + } + std::optional replaced = replacer(expr); + if (replaced.has_value()) { + return *replaced; + } + + cel::Expr new_expr = expr; + switch (new_expr.kind_case()) { + case cel::ExprKindCase::kUnspecifiedExpr: + case cel::ExprKindCase::kConstant: + case cel::ExprKindCase::kIdentExpr: + break; + case cel::ExprKindCase::kSelectExpr: { + cel::SelectExpr& select = new_expr.mutable_select_expr(); + if (select.has_operand()) { + CEL_ASSIGN_OR_RETURN(cel::Expr operand, + CopyAndReplace(select.operand(), replacer, + max_recursion_depth - 1)); + select.set_operand(std::move(operand)); + } + break; + } + case cel::ExprKindCase::kCallExpr: { + cel::CallExpr& call = new_expr.mutable_call_expr(); + if (call.has_target()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr target, + CopyAndReplace(call.target(), replacer, max_recursion_depth - 1)); + call.set_target(std::move(target)); + } + for (auto& arg : call.mutable_args()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_arg, + CopyAndReplace(arg, replacer, max_recursion_depth - 1)); + arg = std::move(new_arg); + } + break; + } + case cel::ExprKindCase::kListExpr: { + cel::ListExpr& list = new_expr.mutable_list_expr(); + for (auto& elem : list.mutable_elements()) { + if (elem.has_expr()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_elem, + CopyAndReplace(elem.expr(), replacer, max_recursion_depth - 1)); + elem.set_expr(std::move(new_elem)); + } + } + break; + } + case cel::ExprKindCase::kStructExpr: { + cel::StructExpr& str = new_expr.mutable_struct_expr(); + for (auto& field : str.mutable_fields()) { + if (field.has_value()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_val, + CopyAndReplace(field.value(), replacer, max_recursion_depth - 1)); + field.set_value(std::move(new_val)); + } + } + break; + } + case cel::ExprKindCase::kMapExpr: { + cel::MapExpr& map = new_expr.mutable_map_expr(); + for (auto& entry : map.mutable_entries()) { + if (entry.has_key()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_key, + CopyAndReplace(entry.key(), replacer, max_recursion_depth - 1)); + entry.set_key(std::move(new_key)); + } + if (entry.has_value()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_val, + CopyAndReplace(entry.value(), replacer, max_recursion_depth - 1)); + entry.set_value(std::move(new_val)); + } + } + break; + } + case cel::ExprKindCase::kComprehensionExpr: { + cel::ComprehensionExpr& comp = new_expr.mutable_comprehension_expr(); + if (comp.has_accu_init()) { + CEL_ASSIGN_OR_RETURN(cel::Expr new_accu_init, + CopyAndReplace(comp.accu_init(), replacer, + max_recursion_depth - 1)); + comp.set_accu_init(std::move(new_accu_init)); + } + if (comp.has_iter_range()) { + CEL_ASSIGN_OR_RETURN(cel::Expr new_iter_range, + CopyAndReplace(comp.iter_range(), replacer, + max_recursion_depth - 1)); + comp.set_iter_range(std::move(new_iter_range)); + } + if (comp.has_loop_condition()) { + CEL_ASSIGN_OR_RETURN(cel::Expr new_loop_condition, + CopyAndReplace(comp.loop_condition(), replacer, + max_recursion_depth - 1)); + comp.set_loop_condition(std::move(new_loop_condition)); + } + if (comp.has_loop_step()) { + CEL_ASSIGN_OR_RETURN(cel::Expr new_loop_step, + CopyAndReplace(comp.loop_step(), replacer, + max_recursion_depth - 1)); + comp.set_loop_step(std::move(new_loop_step)); + } + if (comp.has_result()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_result, + CopyAndReplace(comp.result(), replacer, max_recursion_depth - 1)); + comp.set_result(std::move(new_result)); + } + break; + } + } + return new_expr; +} + +ListNodeBuilder AstFactoryInterface::NewListBuilder( + int64_t id) { + return ListNodeBuilder(id); +} + +StructNodeBuilder AstFactoryInterface::NewStructBuilder( + int64_t id, std::string name) { + return StructNodeBuilder(id, std::move(name)); +} + +MapNodeBuilder AstFactoryInterface::NewMapBuilder( + int64_t id) { + return MapNodeBuilder(id); +} + +} // namespace cel::parser_internal diff --git a/parser/internal/ast_factory.h b/parser/internal/ast_factory.h index c0e8a6b7e..2736b02ec 100644 --- a/parser/internal/ast_factory.h +++ b/parser/internal/ast_factory.h @@ -16,12 +16,12 @@ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ #include +#include #include -#include -#include +#include "absl/functional/function_ref.h" +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "common/constant.h" #include "common/expr.h" #include "common/expr_factory.h" #include "parser/internal/ast_factory_interface.h" @@ -33,21 +33,11 @@ namespace cel::parser_internal { template <> class ListNodeBuilder { public: - explicit ListNodeBuilder(int64_t id) { - expr_.set_id(id); - expr_.mutable_list_expr(); - } - - ListNodeBuilder& Add(cel::Expr element, bool optional = false) { - cel::ListExpr& list_val = expr_.mutable_list_expr(); - cel::ListExprElement expr_element; - expr_element.set_expr(std::move(element)); - expr_element.set_optional(optional); - list_val.mutable_elements().push_back(std::move(expr_element)); - return *this; - } - - cel::Expr Build() { return std::move(expr_); } + explicit ListNodeBuilder(int64_t id); + + ListNodeBuilder& Add(cel::Expr element, bool optional = false); + + cel::Expr Build(); private: cel::Expr expr_; @@ -56,24 +46,12 @@ class ListNodeBuilder { template <> class MapNodeBuilder { public: - explicit MapNodeBuilder(int64_t id) { - expr_.set_id(id); - expr_.mutable_map_expr(); - } + explicit MapNodeBuilder(int64_t id); MapNodeBuilder& Add(int64_t id, cel::Expr key, cel::Expr value, - bool optional = false) { - cel::MapExpr& map_val = expr_.mutable_map_expr(); - cel::MapExprEntry entry; - entry.set_id(id); - entry.set_key(std::move(key)); - entry.set_value(std::move(value)); - entry.set_optional(optional); - map_val.mutable_entries().push_back(std::move(entry)); - return *this; - } - - cel::Expr Build() { return std::move(expr_); } + bool optional = false); + + cel::Expr Build(); private: cel::Expr expr_; @@ -82,24 +60,12 @@ class MapNodeBuilder { template <> class StructNodeBuilder { public: - explicit StructNodeBuilder(int64_t id, std::string name) { - expr_.set_id(id); - expr_.mutable_struct_expr().set_name(std::move(name)); - } + explicit StructNodeBuilder(int64_t id, std::string name); StructNodeBuilder& Add(int64_t id, std::string name, cel::Expr value, - bool optional = false) { - cel::StructExpr& struct_val = expr_.mutable_struct_expr(); - cel::StructExprField field; - field.set_id(id); - field.set_name(std::move(name)); - field.set_value(std::move(value)); - field.set_optional(optional); - struct_val.mutable_fields().push_back(std::move(field)); - return *this; - } - - cel::Expr Build() { return std::move(expr_); } + bool optional = false); + + cel::Expr Build(); private: cel::Expr expr_; @@ -117,34 +83,28 @@ class AstFactoryInterface : public cel::ExprFactory { ~AstFactoryInterface() override = default; // Node inspection and encapsulation API - int64_t GetId(const cel::Expr& expr) const { return expr.id(); } + int64_t GetId(const cel::Expr& expr) const; + + bool IsEmpty(const cel::Expr& expr) const; - bool IsEmpty(const cel::Expr& expr) const { return expr.id() == 0; } + bool IsConst(const cel::Expr& expr) const; - bool IsConst(const cel::Expr& expr) const { return expr.has_const_expr(); } + bool IsIdent(const cel::Expr& expr) const; - bool IsIdent(const cel::Expr& expr) const { return expr.has_ident_expr(); } + absl::string_view GetIdentName(const cel::Expr& expr) const; - absl::string_view GetIdentName(const cel::Expr& expr) const { - return expr.has_ident_expr() ? absl::string_view(expr.ident_expr().name()) - : absl::string_view(); - } + bool IsSelect(const cel::Expr& expr) const; - bool IsSelect(const cel::Expr& expr) const { return expr.has_select_expr(); } + bool IsPresenceTest(const cel::Expr& expr) const; - bool IsPresenceTest(const cel::Expr& expr) const { - return expr.has_select_expr() && expr.select_expr().test_only(); - } + const cel::Expr* GetSelectOperand(const cel::Expr& expr) const; - const cel::Expr* GetSelectOperand(const cel::Expr& expr) const { - return expr.has_select_expr() ? &expr.select_expr().operand() : nullptr; - } + absl::string_view GetSelectField(const cel::Expr& expr) const; - absl::string_view GetSelectField(const cel::Expr& expr) const { - return expr.has_select_expr() - ? absl::string_view(expr.select_expr().field()) - : absl::string_view(); - } + absl::StatusOr CopyAndReplace( + const cel::Expr& expr, + absl::FunctionRef(const cel::Expr&)> replacer, + int max_recursion_depth = 1000) const; // Node creation API using cel::ExprFactory::NewBoolConst; @@ -161,17 +121,11 @@ class AstFactoryInterface : public cel::ExprFactory { using cel::ExprFactory::NewUintConst; using cel::ExprFactory::NewUnspecified; - ListNodeBuilder NewListBuilder(int64_t id) { - return ListNodeBuilder(id); - } + ListNodeBuilder NewListBuilder(int64_t id); - StructNodeBuilder NewStructBuilder(int64_t id, std::string name) { - return StructNodeBuilder(id, std::move(name)); - } + StructNodeBuilder NewStructBuilder(int64_t id, std::string name); - MapNodeBuilder NewMapBuilder(int64_t id) { - return MapNodeBuilder(id); - } + MapNodeBuilder NewMapBuilder(int64_t id); }; using AstFactory = AstFactoryInterface; diff --git a/parser/internal/ast_factory_interface.h b/parser/internal/ast_factory_interface.h index 3b8c9bdfb..c656da22d 100644 --- a/parser/internal/ast_factory_interface.h +++ b/parser/internal/ast_factory_interface.h @@ -16,26 +16,15 @@ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ #include +#include #include #include #include -namespace cel::parser_internal { +#include "absl/functional/function_ref.h" +#include "absl/status/statusor.h" -// Interface for decoupling parser logic from the underlying AST node -// data structures. -// -// By parameterizing the parser and factory on `ExprNode`, alternative AST node -// representations (such as `cel::Expr`) can be constructed without modifying -// parser rules. -// -// To implement AST construction using an alternative AST structure: -// 1. Define or specify your custom node type `MyNode`. -// 2. Implement a concrete factory specialization `AstFactoryInterface` -// that provides inspection (`GetId`, `IsSelect`, etc.) and creation -// (`NewCall`, `NewListBuilder`, etc.) operations for `MyNode`. -// 3. Instantiate the parser worker with your node type: -// `PrattParserWorker`. +namespace cel::parser_internal { template class ListNodeBuilder { @@ -60,6 +49,20 @@ class StructNodeBuilder { ExprNode Build(); }; +// Interface for decoupling parser logic from the underlying AST node +// data structures. +// +// By parameterizing the parser and factory on `ExprNode`, alternative AST node +// representations (such as `cel::Expr`) can be constructed without modifying +// parser rules. +// +// To implement AST construction using an alternative AST structure: +// 1. Define or specify your custom node type `MyNode`. +// 2. Implement a concrete factory specialization `AstFactoryInterface` +// that provides inspection (`GetId`, `IsSelect`, etc.) and creation +// (`NewCall`, `NewListBuilder`, etc.) operations for `MyNode`. +// 3. Instantiate the parser worker with your node type: +// `PrattParserWorker`. template class AstFactoryInterface { public: @@ -78,6 +81,10 @@ class AstFactoryInterface { bool IsPresenceTest(const ExprNode& expr) const; const ExprNode* GetSelectOperand(const ExprNode& expr) const; std::string_view GetSelectField(const ExprNode& expr) const; + absl::StatusOr CopyAndReplace( + const ExprNode& expr, + absl::FunctionRef(const ExprNode&)> replacer, + int max_recursion_depth = 1000) const; ExprNode NewUnspecified(int64_t id); ExprNode NewNullConst(int64_t id); diff --git a/parser/internal/ast_factory_test.cc b/parser/internal/ast_factory_test.cc index a66c8ee6b..3ef08825e 100644 --- a/parser/internal/ast_factory_test.cc +++ b/parser/internal/ast_factory_test.cc @@ -14,17 +14,23 @@ #include "parser/internal/ast_factory.h" -#include +#include +#include #include #include +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" #include "absl/strings/string_view.h" +#include "common/constant.h" #include "common/expr.h" #include "internal/testing.h" namespace cel::parser_internal { namespace { +using ::absl_testing::StatusIs; + TEST(AstFactoryInterfaceTest, AstFactoryUnspecified) { AstFactory factory; cel::Expr expr = factory.NewUnspecified(1); @@ -196,5 +202,196 @@ TEST(AstFactoryInterfaceTest, AstFactoryMap) { EXPECT_TRUE(map_expr.map_expr().entries()[1].optional()); } +TEST(AstFactoryInterfaceTest, CopyAndReplace) { + AstFactory factory; + + // x + 1 + std::vector args; + args.push_back(factory.NewIdent(1, "x")); + args.push_back(factory.NewIntConst(2, 1)); + cel::Expr expr = factory.NewCall(3, "_+_", std::move(args)); + + // Transform: x -> y, 1 -> 2 + ASSERT_OK_AND_ASSIGN( + cel::Expr transformed, + factory.CopyAndReplace( + expr, [&](const cel::Expr& e) -> std::optional { + if (e.has_ident_expr() && e.ident_expr().name() == "x") { + return factory.NewIdent(e.id(), "y"); + } + if (e.has_const_expr() && e.const_expr().has_int_value() && + e.const_expr().int_value() == 1) { + return factory.NewIntConst(e.id(), 2); + } + return std::nullopt; + })); + + // Expected: y + 2 + std::vector expected_args; + expected_args.push_back(factory.NewIdent(1, "y")); + expected_args.push_back(factory.NewIntConst(2, 2)); + cel::Expr expected = factory.NewCall(3, "_+_", std::move(expected_args)); + + EXPECT_EQ(transformed, expected); +} + +TEST(AstFactoryInterfaceTest, CopyAndReplaceDeep) { + AstFactory factory; + + auto replacer = [&](const cel::Expr& e) -> std::optional { + if (e.has_const_expr() && e.const_expr().has_double_value()) { + return factory.NewIntConst( + e.id(), static_cast(e.const_expr().double_value())); + } + return std::nullopt; + }; + + cel::Expr expr = factory.NewCall( + 1, "func", + std::vector{ + factory.NewListBuilder(2) + .Add(factory.NewUnspecified(3)) + .Add(factory.NewNullConst(4)) + .Add(factory.NewBoolConst(5, true)) + .Add(factory.NewIntConst(6, 42)) + .Build(), + factory.NewMapBuilder(7) + .Add(991, factory.NewStringConst(8, "k1"), + factory.NewUintConst(9, 100u)) + .Add(992, factory.NewBytesConst(10, "b1"), + factory.NewDoubleConst(11, 3.14159)) + .Build(), + factory.NewStructBuilder(12, "S") + .Add(32, "f1", factory.NewIdent(13, "x")) + .Add( + 33, "f2", + factory.NewSelect(14, factory.NewIdent(15, "y"), "sel_field")) + .Add(34, "f3", + factory.NewPresenceTest(16, factory.NewIdent(17, "z"), + "pres_field")) + .Build(), + factory.NewMemberCall( + 18, "mem_func", factory.NewIdent(19, "target"), + std::vector{[&]() { + cel::Expr comp_expr; + comp_expr.set_id(20); + auto& comp = comp_expr.mutable_comprehension_expr(); + comp.set_iter_var("i"); + comp.set_iter_var2("i2"); + comp.set_accu_var("a"); + comp.set_accu_init(factory.NewDoubleConst(21, 2.71828)); + comp.set_iter_range(factory.NewIdent(22, "range")); + comp.set_loop_condition(factory.NewBoolConst(23, true)); + comp.set_loop_step(factory.NewCall( + 24, "step_func", + std::vector{factory.NewIdent(25, "accu")})); + comp.set_result(factory.NewIdent(26, "result")); + return comp_expr; + }()})}); + + ASSERT_OK_AND_ASSIGN(cel::Expr transformed_expr, + factory.CopyAndReplace(expr, replacer)); + + cel::Expr expected_transformed_expr = factory.NewCall( + 1, "func", + std::vector{ + factory.NewListBuilder(2) + .Add(factory.NewUnspecified(3)) + .Add(factory.NewNullConst(4)) + .Add(factory.NewBoolConst(5, true)) + .Add(factory.NewIntConst(6, 42)) + .Build(), + factory.NewMapBuilder(7) + .Add(991, factory.NewStringConst(8, "k1"), + factory.NewUintConst(9, 100u)) + .Add(992, factory.NewBytesConst(10, "b1"), + factory.NewIntConst(11, 3)) // 3.14159 -> 3 + .Build(), + factory.NewStructBuilder(12, "S") + .Add(32, "f1", factory.NewIdent(13, "x")) + .Add( + 33, "f2", + factory.NewSelect(14, factory.NewIdent(15, "y"), "sel_field")) + .Add(34, "f3", + factory.NewPresenceTest(16, factory.NewIdent(17, "z"), + "pres_field")) + .Build(), + factory.NewMemberCall( + 18, "mem_func", factory.NewIdent(19, "target"), + std::vector{[&]() { + cel::Expr comp_expr; + comp_expr.set_id(20); + auto& comp = comp_expr.mutable_comprehension_expr(); + comp.set_iter_var("i"); + comp.set_iter_var2("i2"); + comp.set_accu_var("a"); + comp.set_accu_init(factory.NewIntConst(21, 2)); // 2.71828 -> 2 + comp.set_iter_range(factory.NewIdent(22, "range")); + comp.set_loop_condition(factory.NewBoolConst(23, true)); + comp.set_loop_step(factory.NewCall( + 24, "step_func", + std::vector{factory.NewIdent(25, "accu")})); + comp.set_result(factory.NewIdent(26, "result")); + return comp_expr; + }()})}); + + EXPECT_EQ(transformed_expr, expected_transformed_expr); +} + +TEST(AstFactoryInterfaceTest, CopyAndReplacePrune) { + AstFactory factory; + + // (x + 1) + 2 + std::vector inner_args; + inner_args.push_back(factory.NewIdent(1, "x")); + inner_args.push_back(factory.NewIntConst(2, 1)); + cel::Expr inner_expr = factory.NewCall(3, "_+_", std::move(inner_args)); + + std::vector args; + args.push_back(std::move(inner_expr)); + args.push_back(factory.NewIntConst(4, 2)); + cel::Expr expr = factory.NewCall(5, "_+_", std::move(args)); + + // Replace the inner call (id 3) with a single ident "y" (id 9), pruning the + // subtree. + ASSERT_OK_AND_ASSIGN( + cel::Expr transformed, + factory.CopyAndReplace( + expr, [&](const cel::Expr& e) -> std::optional { + if (e.id() == 3) { + return factory.NewIdent(9, "y"); + } + return std::nullopt; + })); + + // Expected: y + 2 + std::vector expected_args; + expected_args.push_back(factory.NewIdent(9, "y")); + expected_args.push_back(factory.NewIntConst(4, 2)); + cel::Expr expected = factory.NewCall(5, "_+_", std::move(expected_args)); + + EXPECT_EQ(transformed, expected); +} + +TEST(AstFactoryInterfaceTest, CopyAndReplaceMaxRecursionDepth) { + AstFactory factory; + + cel::Expr expr = factory.NewIdent(1, "x"); + for (int i = 2; i <= 10; ++i) { + std::vector args; + args.push_back(std::move(expr)); + args.push_back(factory.NewIntConst(i, 1)); + expr = factory.NewCall(i, "_+_", std::move(args)); + } + + EXPECT_THAT(factory.CopyAndReplace( + expr, + [](const cel::Expr&) -> std::optional { + return std::nullopt; + }, + /*max_recursion_depth=*/3), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + } // namespace } // namespace cel::parser_internal From 6cb7d127fb22181c15c84d673021b57fb64af074 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 24 Jul 2026 12:06:36 -0700 Subject: [PATCH 135/350] [Pratt Parser] Add support for macro expansion. PiperOrigin-RevId: 953490682 --- parser/internal/BUILD | 15 + parser/internal/ast_factory.cc | 18 ++ parser/internal/ast_factory.h | 41 ++- parser/internal/ast_factory_interface.h | 19 ++ parser/internal/ast_factory_test.cc | 52 ++++ parser/internal/pratt_parser.cc | 158 +++++------ parser/internal/pratt_parser.h | 34 ++- parser/internal/pratt_parser_test.cc | 355 ++++++++++++++++++++++-- parser/internal/pratt_parser_worker.cc | 8 +- parser/internal/pratt_parser_worker.h | 177 +++++++++++- parser/macro_expr_factory.h | 6 + 11 files changed, 755 insertions(+), 128 deletions(-) diff --git a/parser/internal/BUILD b/parser/internal/BUILD index 535a1bb79..e5593a9e2 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -26,6 +26,7 @@ cc_library( deps = [ "@com_google_absl//absl/functional:function_ref", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/types:span", ], ) @@ -38,10 +39,15 @@ cc_library( "//common:expr", "//common:expr_factory", "//internal:status_macros", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:macro_registry", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:function_ref", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", ], ) @@ -86,6 +92,7 @@ cc_library( "//parser:options", "//parser:parser_interface", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -127,12 +134,17 @@ cc_test( srcs = ["ast_factory_test.cc"], deps = [ ":ast_factory", + ":ast_factory_interface", "//common:constant", "//common:expr", "//internal:testing", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:macro_registry", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", ], ) @@ -159,6 +171,8 @@ cc_test( "//common:source", "//internal:status_macros", "//internal:testing", + "//parser:macro", + "//parser:macro_expr_factory", "//parser:options", "//parser:parser_interface", "//testutil:expr_printer", @@ -169,6 +183,7 @@ cc_test( "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", ], ) diff --git a/parser/internal/ast_factory.cc b/parser/internal/ast_factory.cc index b339f06b4..0c9a7d661 100644 --- a/parser/internal/ast_factory.cc +++ b/parser/internal/ast_factory.cc @@ -14,6 +14,7 @@ #include "parser/internal/ast_factory.h" +#include #include #include #include @@ -25,6 +26,8 @@ #include "absl/strings/string_view.h" #include "common/expr.h" #include "internal/status_macros.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/macro.h" namespace cel::parser_internal { @@ -266,4 +269,19 @@ MapNodeBuilder AstFactoryInterface::NewMapBuilder( return MapNodeBuilder(id); } +std::optional> +AstFactoryInterface::NewMacroExprExpander(std::string_view name, + size_t arg_count, + bool receiver_style) { + if (macro_registry_ == nullptr) { + return std::nullopt; + } + std::optional macro = + macro_registry_->FindMacro(name, arg_count, receiver_style); + if (!macro) { + return std::nullopt; + } + return std::optional>(std::in_place, *macro); +} + } // namespace cel::parser_internal diff --git a/parser/internal/ast_factory.h b/parser/internal/ast_factory.h index 2736b02ec..6240ce527 100644 --- a/parser/internal/ast_factory.h +++ b/parser/internal/ast_factory.h @@ -15,16 +15,24 @@ #ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_H_ +#include #include +#include #include #include +#include +#include "absl/base/nullability.h" #include "absl/functional/function_ref.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "common/expr.h" #include "common/expr_factory.h" #include "parser/internal/ast_factory_interface.h" +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/macro_registry.h" namespace cel::parser_internal { @@ -71,10 +79,35 @@ class StructNodeBuilder { cel::Expr expr_; }; +template <> +class AstFactoryInterface; + +template <> +class MacroExprExpanderSupport : public cel::MacroExprFactory {}; + +template <> +class MacroExprExpander { + public: + explicit MacroExprExpander(cel::Macro macro) : macro_(std::move(macro)) {} + + std::optional Expand( + std::optional> target, + absl::Span args, + MacroExprExpanderSupport& support) { + return macro_.Expand(support, target, args); + } + + private: + cel::Macro macro_; +}; + template <> class AstFactoryInterface : public cel::ExprFactory { public: - AstFactoryInterface() = default; + explicit AstFactoryInterface( + const cel::MacroRegistry* absl_nullable macro_registry = nullptr) + : macro_registry_(macro_registry) {} + AstFactoryInterface(const AstFactoryInterface&) = delete; AstFactoryInterface(AstFactoryInterface&&) = delete; AstFactoryInterface& operator=(const AstFactoryInterface&) = delete; @@ -126,6 +159,12 @@ class AstFactoryInterface : public cel::ExprFactory { StructNodeBuilder NewStructBuilder(int64_t id, std::string name); MapNodeBuilder NewMapBuilder(int64_t id); + + std::optional> NewMacroExprExpander( + std::string_view name, size_t arg_count, bool receiver_style); + + private: + const cel::MacroRegistry* absl_nullable macro_registry_ = nullptr; }; using AstFactory = AstFactoryInterface; diff --git a/parser/internal/ast_factory_interface.h b/parser/internal/ast_factory_interface.h index c656da22d..dbfa02a0f 100644 --- a/parser/internal/ast_factory_interface.h +++ b/parser/internal/ast_factory_interface.h @@ -15,7 +15,9 @@ #ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_AST_FACTORY_INTERFACE_H_ +#include #include +#include #include #include #include @@ -23,6 +25,7 @@ #include "absl/functional/function_ref.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" namespace cel::parser_internal { @@ -49,6 +52,17 @@ class StructNodeBuilder { ExprNode Build(); }; +template +class MacroExprExpanderSupport {}; + +template +class MacroExprExpander { + public: + std::optional Expand( + std::optional> target, + absl::Span args, MacroExprExpanderSupport& support); +}; + // Interface for decoupling parser logic from the underlying AST node // data structures. // @@ -104,6 +118,11 @@ class AstFactoryInterface { ListNodeBuilder NewListBuilder(int64_t id); MapNodeBuilder NewMapBuilder(int64_t id); StructNodeBuilder NewStructBuilder(int64_t id, std::string name); + + // Returns a macro expander for the given macro name, or null if there + // is no registered macro with that name and argument count. + std::optional> NewMacroExprExpander( + std::string_view name, size_t arg_count, bool receiver_style); }; } // namespace cel::parser_internal diff --git a/parser/internal/ast_factory_test.cc b/parser/internal/ast_factory_test.cc index 3ef08825e..3e80a4a17 100644 --- a/parser/internal/ast_factory_test.cc +++ b/parser/internal/ast_factory_test.cc @@ -22,13 +22,20 @@ #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "common/constant.h" #include "common/expr.h" #include "internal/testing.h" +#include "parser/internal/ast_factory_interface.h" +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/macro_registry.h" namespace cel::parser_internal { namespace { +using ::absl_testing::IsOk; + using ::absl_testing::StatusIs; TEST(AstFactoryInterfaceTest, AstFactoryUnspecified) { @@ -393,5 +400,50 @@ TEST(AstFactoryInterfaceTest, CopyAndReplaceMaxRecursionDepth) { StatusIs(absl::StatusCode::kInvalidArgument)); } +class TestMacroExprExpanderSupport + : public MacroExprExpanderSupport { + public: + int64_t NextId() override { return 42; } + int64_t CopyId(int64_t id) override { return id; } + cel::Expr ReportError(std::string_view) override { return cel::Expr(); } + cel::Expr ReportErrorAt(const cel::Expr&, std::string_view) override { + return cel::Expr(); + } +}; + +TEST(AstFactoryInterfaceTest, MacroExprExpander) { + MacroRegistry macro_registry; + AstFactory factory(¯o_registry); + ASSERT_OK_AND_ASSIGN( + auto foo_macro, + Macro::Global("foo", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.NewCall("my_macro", std::move(args)); + })); + + ASSERT_THAT(macro_registry.RegisterMacro(foo_macro), IsOk()); + + auto expander1 = factory.NewMacroExprExpander("foo", 1, false); + ASSERT_TRUE(expander1.has_value()); + + std::vector expand_args; + expand_args.push_back(factory.NewIdent(1, "x")); + + TestMacroExprExpanderSupport support; + auto result = + expander1->Expand(std::nullopt, absl::MakeSpan(expand_args), support); + ASSERT_TRUE(result.has_value()); + + std::vector expected_args; + expected_args.push_back(factory.NewIdent(1, "x")); + Expr expected = factory.NewCall(42, "my_macro", std::move(expected_args)); + + EXPECT_EQ(*result, expected); + + auto expander2 = factory.NewMacroExprExpander("bar", 1, false); + EXPECT_FALSE(expander2.has_value()); +} + } // namespace } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc index f86974a36..c321fa7b8 100644 --- a/parser/internal/pratt_parser.cc +++ b/parser/internal/pratt_parser.cc @@ -65,104 +65,91 @@ std::string FormatIssues(const cel::Source& source, }); } -class PrattParserBuilderImpl final : public cel::ParserBuilder { - public: - explicit PrattParserBuilderImpl(const cel::ParserOptions& options) - : options_(options) {} - - cel::ParserOptions& GetOptions() override { return options_; } - - absl::Status AddMacro(const cel::Macro& macro) override { - for (const cel::Macro& existing_macro : macros_) { - if (existing_macro.key() == macro.key()) { - return absl::AlreadyExistsError( - absl::StrCat("macro already exists: ", macro.key())); - } - } - macros_.push_back(macro); - return absl::OkStatus(); - } +} // namespace - absl::Status AddLibrary(cel::ParserLibrary library) override { - if (!library.id.empty()) { - auto [it, inserted] = library_ids_.insert(library.id); - if (!inserted) { - return absl::AlreadyExistsError( - absl::StrCat("parser library already exists: ", library.id)); - } +absl::Status PrattParserBuilderImpl::AddMacro(const cel::Macro& macro) { + for (const cel::Macro& existing_macro : macros_) { + if (existing_macro.key() == macro.key()) { + return absl::AlreadyExistsError( + absl::StrCat("macro already exists: ", macro.key())); } - libraries_.push_back(std::move(library)); - return absl::OkStatus(); } + macros_.push_back(macro); + return absl::OkStatus(); +} - absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override { - if (subset.library_id.empty()) { - return absl::InvalidArgumentError("subset must have a library id"); - } - std::string library_id = subset.library_id; - auto [it, inserted] = - library_subsets_.insert({library_id, std::move(subset)}); +absl::Status PrattParserBuilderImpl::AddLibrary(cel::ParserLibrary library) { + if (!library.id.empty()) { + auto [it, inserted] = library_ids_.insert(library.id); if (!inserted) { return absl::AlreadyExistsError( - absl::StrCat("parser library subset already exists: ", library_id)); + absl::StrCat("parser library already exists: ", library.id)); } - return absl::OkStatus(); } + libraries_.push_back(std::move(library)); + return absl::OkStatus(); +} + +absl::Status PrattParserBuilderImpl::AddLibrarySubset( + cel::ParserLibrarySubset subset) { + if (subset.library_id.empty()) { + return absl::InvalidArgumentError("subset must have a library id"); + } + std::string library_id = subset.library_id; + auto [it, inserted] = + library_subsets_.insert({library_id, std::move(subset)}); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library subset already exists: ", library_id)); + } + return absl::OkStatus(); +} - absl::StatusOr> Build() override { - using std::swap; - std::vector individual_macros; - swap(individual_macros, macros_); - absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); - - cel::MacroRegistry macro_registry; - - for (const cel::ParserLibrary& library : libraries_) { - CEL_RETURN_IF_ERROR(library.configure(*this)); - if (!library.id.empty()) { - auto it = library_subsets_.find(library.id); - if (it != library_subsets_.end()) { - const cel::ParserLibrarySubset& subset = it->second; - for (const cel::Macro& macro : macros_) { - if (subset.should_include_macro(macro)) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); - } +absl::StatusOr> PrattParserBuilderImpl::Build() { + using std::swap; + std::vector individual_macros; + swap(individual_macros, macros_); + absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); + + cel::MacroRegistry macro_registry; + + for (const cel::ParserLibrary& library : libraries_) { + CEL_RETURN_IF_ERROR(library.configure(*this)); + if (!library.id.empty()) { + auto it = library_subsets_.find(library.id); + if (it != library_subsets_.end()) { + const cel::ParserLibrarySubset& subset = it->second; + for (const cel::Macro& macro : macros_) { + if (subset.should_include_macro(macro)) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); } - macros_.clear(); - continue; } + macros_.clear(); + continue; } - - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); - macros_.clear(); } - absl::flat_hash_set library_ids(library_ids_); + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); + macros_.clear(); + } - if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); - library_ids.insert("stdlib"); - } + absl::flat_hash_set library_ids(library_ids_); - if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); - library_ids.insert("optional"); - } - - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); - return std::make_unique( - options_, std::move(macro_registry), std::move(library_ids)); + if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); + library_ids.insert("stdlib"); } - cel::ParserOptions options_; - std::vector macros_; - absl::flat_hash_set library_ids_; - std::vector libraries_; - absl::flat_hash_map library_subsets_; -}; + if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); + library_ids.insert("optional"); + } -} // namespace + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); + return std::make_unique(options_, std::move(macro_registry), + std::move(library_ids)); +} template class PrattParserWorker; @@ -175,7 +162,8 @@ absl::StatusOr> PrattParserImpl::ParseImpl( source.content().size(), options_.expression_size_codepoint_limit)); } std::vector issues; - PrattParserWorker worker(source, options_, &issues); + AstFactory factory(¯o_registry_); + PrattParserWorker worker(source, options_, &issues, factory); Expr expr = worker.Parse(); if (worker.is_recursion_limit_exceeded()) { return absl::CancelledError( @@ -195,10 +183,11 @@ absl::StatusOr> PrattParserImpl::ParseImpl( for (const auto& [id, pos] : worker.GetNodePositions()) { source_info.mutable_positions().insert({id, pos}); } - source_info.mutable_line_offsets().reserve(worker.GetLineOffsets().size()); - for (int32_t offset : worker.GetLineOffsets()) { + source_info.mutable_line_offsets().reserve(source.line_offsets().size()); + for (int32_t offset : source.line_offsets()) { source_info.mutable_line_offsets().push_back(offset); } + source_info.mutable_macro_calls() = worker.ReleaseMacroCalls(); return std::make_unique(std::move(expr), std::move(source_info)); } @@ -209,9 +198,4 @@ std::unique_ptr PrattParserImpl::ToBuilder() const { return ins; } -std::unique_ptr NewPrattParserBuilder( - const cel::ParserOptions& options) { - return std::make_unique(options); -} - } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser.h b/parser/internal/pratt_parser.h index c39c68521..21e9878f3 100644 --- a/parser/internal/pratt_parser.h +++ b/parser/internal/pratt_parser.h @@ -21,10 +21,13 @@ #include #include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "common/ast.h" #include "common/source.h" +#include "parser/macro.h" #include "parser/macro_registry.h" #include "parser/options.h" #include "parser/parser_interface.h" @@ -64,8 +67,35 @@ class PrattParserImpl final : public cel::Parser { absl::flat_hash_set library_ids_; }; -std::unique_ptr NewPrattParserBuilder( - const cel::ParserOptions& options = cel::ParserOptions()); +class PrattParserBuilderImpl final : public cel::ParserBuilder { + public: + explicit PrattParserBuilderImpl(const cel::ParserOptions& options) + : options_(options) {} + + cel::ParserOptions& GetOptions() override { return options_; } + + absl::Status AddMacro(const cel::Macro& macro) override; + + absl::Status AddLibrary(cel::ParserLibrary library) override; + + absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override; + + absl::StatusOr> Build() override; + + private: + friend class PrattParserImpl; + + cel::ParserOptions options_; + std::vector macros_; + absl::flat_hash_set library_ids_; + std::vector libraries_; + absl::flat_hash_map library_subsets_; +}; + +inline std::unique_ptr NewPrattParserBuilder( + const cel::ParserOptions& options = cel::ParserOptions()) { + return std::make_unique(options); +} } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index b1dd81fbb..708936a4e 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -15,8 +15,11 @@ #include "parser/internal/pratt_parser.h" #include +#include #include +#include #include +#include #include #include "absl/algorithm/container.h" @@ -29,6 +32,7 @@ #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "common/ast.h" #include "common/constant.h" #include "common/expr.h" @@ -37,6 +41,8 @@ #include "internal/testing.h" #include "parser/internal/lexer.h" #include "parser/internal/pratt_parser_worker.h" +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" #include "parser/options.h" #include "parser/parser_interface.h" #include "testutil/expr_printer.h" @@ -47,11 +53,19 @@ namespace cel::parser_internal { namespace { +using ::absl_testing::IsOk; using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::testing::Eq; using ::testing::NotNull; +template +std::string TestName(const testing::TestParamInfo& test_info) { + std::string name = absl::StrCat(test_info.index, "-", test_info.param.source); + absl::c_replace_if(name, [](char c) { return !absl::ascii_isalnum(c); }, '_'); + return name; +} + absl::StatusOr> Parse( std::string_view expression, const cel::ParserOptions& options = cel::ParserOptions(), @@ -76,18 +90,6 @@ struct TestCase { class PrattParserTest : public testing::TestWithParam {}; -std::string Unindent(std::string_view multiline) { - std::vector unindented_lines; - int indent = -1; - for (std::string_view line : absl::StrSplit(multiline, '\n')) { - std::size_t pos = line.find_first_not_of(" \t"); - if (pos == std::string_view::npos) continue; - if (indent == -1) indent = pos; - unindented_lines.push_back(std::string(line.substr(indent))); - } - return absl::StrJoin(unindented_lines, "\n"); -} - std::string_view ConstantKind(const cel::Constant& c) { switch (c.kind_case()) { case ConstantKindCase::kBool: @@ -153,6 +155,31 @@ class KindAndIdAdorner : public cel::test::ExpressionAdorner { } }; +std::string Unindent(std::string_view multiline) { + std::vector unindented_lines; + int indent = -1; + for (std::string_view line : absl::StrSplit(multiline, '\n')) { + std::size_t pos = line.find_first_not_of(" \t"); + if (pos == std::string_view::npos) continue; + if (indent == -1) indent = pos; + unindented_lines.push_back(std::string(line.substr(indent))); + } + return absl::StrJoin(unindented_lines, "\n"); +} + +MATCHER_P(AstIs, expected_ast, "") { + KindAndIdAdorner kind_and_id_adorner; + test::ExprPrinter printer(kind_and_id_adorner); + std::string actual = Unindent(printer.Print(arg)); + std::string expected = Unindent(expected_ast); + if (actual == expected) { + return true; + } + *result_listener << "\n Actual: " << actual + << "\n Expected: " << expected; + return false; +} + TEST_P(PrattParserTest, Parse) { const TestCase& test_case = GetParam(); cel::ParserOptions options; @@ -161,10 +188,7 @@ TEST_P(PrattParserTest, Parse) { test_case.enable_variadic_logical_operators; ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, Parse(test_case.source, options)); - const Expr& root = ast->root_expr(); - KindAndIdAdorner kind_and_id_adorner; - test::ExprPrinter printer(kind_and_id_adorner); - EXPECT_EQ(Unindent(printer.Print(root)), Unindent(test_case.expected_ast)); + EXPECT_THAT(ast->root_expr(), AstIs(test_case.expected_ast)); } std::vector GetParserTestCases() { @@ -1012,14 +1036,9 @@ std::vector GetParserTestCases() { }; } -std::string TestName(const testing::TestParamInfo& test_info) { - std::string name = absl::StrCat(test_info.index, "-", test_info.param.source); - absl::c_replace_if(name, [](char c) { return !absl::ascii_isalnum(c); }, '_'); - return name; -} - INSTANTIATE_TEST_SUITE_P(PrattParserTest, PrattParserTest, - testing::ValuesIn(GetParserTestCases()), TestName); + testing::ValuesIn(GetParserTestCases()), + TestName); struct ErrorTestCase { std::string_view source; @@ -1375,7 +1394,8 @@ std::vector GetErrorTestCases() { } INSTANTIATE_TEST_SUITE_P(PrattParserErrorTest, PrattParserErrorTest, - testing::ValuesIn(GetErrorTestCases())); + testing::ValuesIn(GetErrorTestCases()), + TestName); TEST(PrattParserTest, SourceInfoPositionsPopulated) { ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, Parse("a + b")); @@ -1430,5 +1450,292 @@ TEST(ParserWorkerTest, GetTokenTextBoundsChecking) { ""); } +struct MacroTestCase { + std::string_view source; + std::string_view expected_ast; +}; + +class PrattParserMacroTest : public testing::TestWithParam {}; + +TEST_P(PrattParserMacroTest, MacroExprExpander) { + const MacroTestCase& test_case = GetParam(); + auto builder = NewPrattParserBuilder(); + ASSERT_OK_AND_ASSIGN( + auto global_macro, + Macro::Global("foo", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.NewCall("my_macro", std::move(args)); + })); + + ASSERT_OK_AND_ASSIGN( + auto receiver_macro, + Macro::Receiver("bar", 2, + [](MacroExprFactory& macro_factory, Expr& target, + absl::Span args) -> std::optional { + return macro_factory.NewMemberCall( + "my_bar", std::move(target), std::move(args)); + })); + + ASSERT_THAT(builder->AddMacro(global_macro), IsOk()); + ASSERT_THAT(builder->AddMacro(receiver_macro), IsOk()); + ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.source)); + ASSERT_OK_AND_ASSIGN(auto ast, parser->Parse(*source)); + + EXPECT_THAT(ast->root_expr(), AstIs(test_case.expected_ast)); +} + +std::vector GetMacroTestCases() { + return { + MacroTestCase{ + .source = "foo(x)", + .expected_ast = R"( + my_macro( + x^#2:Expr.Ident# + )^#3:Expr.Call# + )", + }, + MacroTestCase{ + .source = "x.bar(y, z)", + .expected_ast = R"( + x^#1:Expr.Ident#.my_bar( + y^#3:Expr.Ident#, + z^#4:Expr.Ident# + )^#5:Expr.Call# + )", + }, + // Number of args doesn't match the macro definition + MacroTestCase{ + .source = "foo(x, y)", + .expected_ast = R"( + foo( + x^#2:Expr.Ident#, + y^#3:Expr.Ident# + )^#1:Expr.Call# + )", + }, + // Number of args doesn't match the macro definition + MacroTestCase{ + .source = "x.bar(y)", + .expected_ast = R"( + x^#1:Expr.Ident#.bar( + y^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + // No target provided for receiver macro + MacroTestCase{ + .source = "bar(x, y)", + .expected_ast = R"( + bar( + x^#2:Expr.Ident#, + y^#3:Expr.Ident# + )^#1:Expr.Call# + )", + }, + // Target provided for global macro + MacroTestCase{ + .source = "x.foo(y)", + .expected_ast = R"( + x^#1:Expr.Ident#.foo( + y^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + // Global macro not registered + MacroTestCase{ + .source = "baz(x)", + .expected_ast = R"( + baz( + x^#2:Expr.Ident# + )^#1:Expr.Call# + )", + }, + // Receiver macro not registered + MacroTestCase{ + .source = "x.baz(x)", + .expected_ast = R"( + x^#1:Expr.Ident#.baz( + x^#3:Expr.Ident# + )^#2:Expr.Call# + )", + }, + // has() macro + MacroTestCase{ + .source = "has(message.field)", + .expected_ast = R"( + message^#2:Expr.Ident#.field~test-only~^#4:Expr.Select# + )", + }, + }; +} + +INSTANTIATE_TEST_SUITE_P(PrattParserMacroTest, PrattParserMacroTest, + testing::ValuesIn(GetMacroTestCases()), + TestName); + +TEST(PrattParserMacroErrorTest, ReportError) { + auto builder = NewPrattParserBuilder(); + ASSERT_OK_AND_ASSIGN( + auto error_macro, + Macro::Global("bad_macro", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.ReportError("custom macro error"); + })); + + ASSERT_THAT(builder->AddMacro(error_macro), IsOk()); + ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("42 + bad_macro(x)")); + std::vector issues; + auto ast = parser->Parse(*source, &issues); + EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :1:6: custom macro error\n" + " | 42 + bad_macro(x)\n" + " | .....^"); +} + +TEST(PrattParserMacroErrorTest, ReportErrorAt) { + auto builder = NewPrattParserBuilder(); + ASSERT_OK_AND_ASSIGN( + auto error_at_macro, + Macro::Global("bad_macro_at", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.ReportErrorAt(args[0], + "custom error at arg"); + })); + + ASSERT_THAT(builder->AddMacro(error_at_macro), IsOk()); + ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("bad_macro_at(x)")); + std::vector issues; + auto ast = parser->Parse(*source, &issues); + EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :1:14: custom error at arg\n" + " | bad_macro_at(x)\n" + " | .............^"); +} + +TEST(PrattParserMacroCallsTest, MacroCallsDisabledByDefault) { + cel::ParserOptions options; + options.add_macro_calls = false; + ASSERT_OK_AND_ASSIGN(auto ast, Parse("has(a.b)", options)); + EXPECT_TRUE(ast->source_info().macro_calls().empty()); +} + +TEST(PrattParserMacroCallsTest, GlobalMacroCallRecorded) { + cel::ParserOptions options; + options.add_macro_calls = true; + ASSERT_OK_AND_ASSIGN(auto ast, Parse("has(a.b)", options)); + + const auto& macro_calls = ast->source_info().macro_calls(); + EXPECT_FALSE(macro_calls.empty()); + EXPECT_TRUE(macro_calls.contains(ast->root_expr().id())); + + const auto& macro_call = macro_calls.at(ast->root_expr().id()); + EXPECT_THAT(macro_call, AstIs(R"( + has( + a^#2:Expr.Ident#.b^#3:Expr.Select# + )^#0:Expr.Call# + )")); +} + +TEST(PrattParserMacroCallsTest, ReceiverMacroCallRecorded) { + cel::ParserOptions options; + options.add_macro_calls = true; + ASSERT_OK_AND_ASSIGN(auto ast, Parse("[1, 2].exists(x, x > 0)", options)); + + const auto& macro_calls = ast->source_info().macro_calls(); + EXPECT_FALSE(macro_calls.empty()); + EXPECT_TRUE(macro_calls.contains(ast->root_expr().id())); + + const auto& exists_macro_call = macro_calls.at(ast->root_expr().id()); + EXPECT_THAT(exists_macro_call, AstIs(R"( + [ + 1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList#.exists( + x^#5:Expr.Ident#, + _>_( + x^#6:Expr.Ident#, + 0^#8:int64# + )^#7:Expr.Call# + )^#0:Expr.Call# + )")); +} + +TEST(PrattParserMacroCallsTest, NestedMacroCallsUseCopyAndReplaceReplacer) { + cel::ParserOptions options; + options.add_macro_calls = true; + ASSERT_OK_AND_ASSIGN(auto ast, Parse("[1, 2].all(x, has(x.b))", options)); + + const auto& root_expr = ast->root_expr(); + EXPECT_THAT(root_expr, AstIs(R"( + __comprehension__( + // Variable + x, + // Target + [ + 1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList#, + // Accumulator + @result, + // Init + true^#10:bool#, + // LoopCondition + @not_strictly_false( + @result^#11:Expr.Ident# + )^#12:Expr.Call#, + // LoopStep + _&&_( + @result^#13:Expr.Ident#, + x^#7:Expr.Ident#.b~test-only~^#9:Expr.Select# + )^#14:Expr.Call#, + // Result + @result^#15:Expr.Ident#)^#16:Expr.Comprehension# + )")); + + const auto& macro_calls = ast->source_info().macro_calls(); + // There should be 2 recorded macro calls: one for 'all', one for 'has'. + EXPECT_EQ(macro_calls.size(), 2); + + int64_t all_macro_id = root_expr.id(); + EXPECT_TRUE(macro_calls.contains(all_macro_id)); + + const auto& all_macro_call = macro_calls.at(all_macro_id); + // The second argument of 'all' is the inner 'has(x.b)' call. + // Because 'has(x.b)' was already expanded and recorded in macro_calls + // it is represented as an UnspecifiedExpr holding the inner macro's ID. + EXPECT_THAT(all_macro_call, AstIs(R"( + [ + 1^#2:int64#, + 2^#3:int64# + ]^#1:Expr.CreateList#.all( + x^#5:Expr.Ident#, + ^#9:unspecified_expr# + )^#0:Expr.Call# + )")); + + const auto& has_call = all_macro_call.call_expr().args()[1]; + int64_t has_macro_id = has_call.id(); + EXPECT_EQ(has_macro_id, 9); // ^#9:unspecified_expr# + EXPECT_TRUE(macro_calls.contains(has_macro_id)); + + const auto& has_macro_call = macro_calls.at(has_macro_id); + EXPECT_THAT(has_macro_call, AstIs(R"( + has( + x^#7:Expr.Ident#.b^#8:Expr.Select# + )^#0:Expr.Call# + )")); +} + } // namespace } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index ff60e7962..4e40adc1e 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -16,11 +16,11 @@ #include #include -#include #include #include "absl/base/nullability.h" #include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" #include "common/source.h" #include "parser/internal/lexer.h" #include "parser/options.h" @@ -70,7 +70,7 @@ Token ParserWorker::NextToken() { return current_token_; } -bool ParserWorker::Expect(TokenType type, std::string_view msg) { +bool ParserWorker::Expect(TokenType type, absl::string_view msg) { if (peek_token_.type == type) { NextToken(); return true; @@ -139,7 +139,7 @@ void ParserWorker::EraseId(int64_t id) { } } -void ParserWorker::ReportError(int32_t position, std::string_view msg) { +void ParserWorker::ReportError(int32_t position, absl::string_view msg) { cel::SourceLocation loc; if (auto found = source_.GetLocation(position); found.has_value()) { loc = *found; @@ -148,7 +148,7 @@ void ParserWorker::ReportError(int32_t position, std::string_view msg) { } void ParserWorker::ReportError(const SourceLocation& loc, - std::string_view msg) { + absl::string_view msg) { error_count_++; if (parse_issues_ != nullptr) { parse_issues_->push_back(cel::ParseIssue(loc, std::string(msg))); diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index a510f32a3..45221714f 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include "absl/base/nullability.h" +#include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_map.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" @@ -114,16 +116,60 @@ class ParserWorker { template class PrattParserWorker : public ParserWorker { public: + using ParserWorker::NextId; + explicit PrattParserWorker( const cel::Source& source, const cel::ParserOptions& options, - std::vector* absl_nullable parse_issues) - : ParserWorker(source, options, parse_issues) { + std::vector* absl_nullable parse_issues, + AstFactoryInterface& ast_factory) + : ParserWorker(source, options, parse_issues), ast_factory_(ast_factory) { this->InitTokenStream(); } ExprNode Parse(); + absl::flat_hash_map ReleaseMacroCalls() { + return std::move(macro_calls_); + } + private: + class MacroExpanderSupport : public MacroExprExpanderSupport { + public: + MacroExpanderSupport(PrattParserWorker& worker, int32_t macro_position) + : worker_(worker), macro_position_(macro_position) {} + + int64_t NextId() { + return macro_position_ >= 0 ? worker_.NextId(macro_position_) + : worker_.NextId(); + } + + int64_t CopyId(int64_t id) { return worker_.CopyId(id); } + + ExprNode ReportError(absl::string_view message) { + if (macro_position_ >= 0) { + worker_.ReportError(macro_position_, message); + } else { + worker_.ReportError(worker_.current_token_.start, message); + } + return ExprNode(); + } + + ExprNode ReportErrorAt(const ExprNode& expr, absl::string_view message) { + int32_t pos = 0; + auto it = + worker_.GetNodePositions().find(worker_.ast_factory_.GetId(expr)); + if (it != worker_.GetNodePositions().end()) { + pos = it->second; + } + worker_.ReportError(pos, message); + return ExprNode(); + } + + private: + PrattParserWorker& worker_; + int32_t macro_position_; + }; + using CelOperator = ::google::api::expr::common::CelOperator; ExprNode ParseExpr(); @@ -151,7 +197,18 @@ class PrattParserWorker : public ParserWorker { ExprNode BalanceLogical(absl::string_view op, std::vector terms, std::vector ops, bool enable_variadic); - AstFactoryInterface ast_factory_; + std::optional TryExpandMacro(int64_t expr_id, + absl::string_view function, + ExprNode* target, + std::vector& args); + + // Save the original structure of the expression before macro expansion. + void RecordMacroCall(int64_t macro_id, absl::string_view function, + std::optional target, + std::vector arguments); + + AstFactoryInterface& ast_factory_; + absl::flat_hash_map macro_calls_; }; template @@ -172,28 +229,26 @@ ExprNode PrattParserWorker::ParseExpr() { if (recursion_limit_exceeded_) { return ExprNode(); } - if (recursion_depth_ >= options_.max_recursion_depth) { + if (recursion_depth_ > options_.max_recursion_depth) { recursion_limit_exceeded_ = true; return ExprNode(); } recursion_depth_++; + auto recursion_cleanup = absl::MakeCleanup([this]() { recursion_depth_--; }); ExprNode lhs = ParseConditionalOr(); if (peek_token_.type == TokenType::kQuestion) { NextToken(); int64_t op_id = NextId(); ExprNode true_expr = ParseConditionalOr(); if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { - recursion_depth_--; return lhs; } ExprNode false_expr = ParseExpr(); - recursion_depth_--; return ast_factory_.NewCall( op_id, CelOperator::CONDITIONAL, std::vector{std::move(lhs), std::move(true_expr), std::move(false_expr)}); } - recursion_depth_--; return lhs; } @@ -417,8 +472,14 @@ ExprNode PrattParserWorker::ParseMember() { Token lparen = NextToken(); int64_t call_id = NextId(lparen); std::vector args = ParseArguments(TokenType::kRightParen); - lhs = ast_factory_.NewMemberCall(call_id, id_text, std::move(lhs), - std::move(args)); + if (std::optional expanded = + TryExpandMacro(call_id, id_text, &lhs, args); + expanded.has_value()) { + lhs = std::move(*expanded); + } else { + lhs = ast_factory_.NewMemberCall(call_id, id_text, std::move(lhs), + std::move(args)); + } } else { lhs = ast_factory_.NewSelect(NextId(dot_tok), std::move(lhs), id_text); } @@ -509,7 +570,12 @@ ExprNode PrattParserWorker::ParsePrimary() { if (peek_token_.type == TokenType::kLeftParen) { NextToken(); std::vector args = ParseArguments(TokenType::kRightParen); - expr = ast_factory_.NewCall(id, name, std::move(args)); + if (auto expanded = TryExpandMacro(id, name, nullptr, args); + expanded.has_value()) { + expr = std::move(*expanded); + } else { + expr = ast_factory_.NewCall(id, name, std::move(args)); + } } else { expr = ast_factory_.NewIdent(id, std::move(name)); } @@ -855,6 +921,97 @@ ExprNode PrattParserWorker::BalanceLogical( return BalancedTree(op, terms, ops, 0, ops.size() - 1); } +template +std::optional PrattParserWorker::TryExpandMacro( + int64_t expr_id, absl::string_view function, ExprNode* target, + std::vector& args) { + bool is_receiver = target != nullptr; + auto expander = + ast_factory_.NewMacroExprExpander(function, args.size(), is_receiver); + if (!expander) { + return std::nullopt; + } + + std::vector macro_args; + ExprNode macro_target; + bool add_macro_calls = options_.add_macro_calls; + // We must build the copies of the macro arguments before calling Expand, + // because Expand is allowed to mutate the arguments in-place upon success. + // However, we only record the macro call if Expand actually succeeds. + if (add_macro_calls) { + auto build_macro_call_arg = [&](const ExprNode& expr) -> ExprNode { + absl::StatusOr copy_or = ast_factory_.CopyAndReplace( + expr, + [this](const ExprNode& e) -> std::optional { + if (auto it = macro_calls_.find(ast_factory_.GetId(e)); + it != macro_calls_.end()) { + return ast_factory_.NewUnspecified(ast_factory_.GetId(e)); + } + return std::nullopt; + }, + options_.max_recursion_depth - recursion_depth_); + if (!copy_or.ok()) { + int32_t macro_position = 0; + if (auto it = positions_.find(expr_id); it != positions_.end()) { + macro_position = it->second; + } + ReportError(macro_position, copy_or.status().message()); + return ast_factory_.NewUnspecified(0); + } + return *std::move(copy_or); + }; + macro_args.reserve(args.size()); + for (const auto& arg : args) { + macro_args.push_back(build_macro_call_arg(arg)); + } + if (target != nullptr) { + macro_target = build_macro_call_arg(*target); + } + } + + absl::optional> target_ref; + if (target != nullptr) { + target_ref = *target; + } + + int32_t macro_position = 0; + if (auto it = positions_.find(expr_id); it != positions_.end()) { + macro_position = it->second; + } + MacroExpanderSupport support(*this, macro_position); + std::optional expanded_expr = + expander->Expand(target_ref, absl::MakeSpan(args), support); + + if (expanded_expr) { + if (add_macro_calls) { + RecordMacroCall(ast_factory_.GetId(*expanded_expr), function, + target != nullptr + ? std::make_optional(std::move(macro_target)) + : std::nullopt, + std::move(macro_args)); + } + EraseId(expr_id); + return expanded_expr; + } + + return std::nullopt; +} + +template +void PrattParserWorker::RecordMacroCall( + int64_t macro_id, absl::string_view function, + std::optional target, std::vector arguments) { + ExprNode call_expr; + if (target.has_value()) { + call_expr = ast_factory_.NewMemberCall( + 0, std::string(function), std::move(*target), std::move(arguments)); + } else { + call_expr = + ast_factory_.NewCall(0, std::string(function), std::move(arguments)); + } + macro_calls_.insert({macro_id, std::move(call_expr)}); +} + } // namespace cel::parser_internal #endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ diff --git a/parser/macro_expr_factory.h b/parser/macro_expr_factory.h index c66aa4fe0..75f8d1d74 100644 --- a/parser/macro_expr_factory.h +++ b/parser/macro_expr_factory.h @@ -33,6 +33,11 @@ namespace cel { class ParserMacroExprFactory; class TestMacroExprFactory; +namespace parser_internal { +template +class MacroExprExpanderSupport; +} // namespace parser_internal + // `MacroExprFactory` is a specialization of `ExprFactory` for `MacroExpander` // which disallows explicitly specifying IDs. class MacroExprFactory : protected ExprFactory { @@ -318,6 +323,7 @@ class MacroExprFactory : protected ExprFactory { private: friend class ParserMacroExprFactory; friend class TestMacroExprFactory; + friend class parser_internal::MacroExprExpanderSupport; explicit MacroExprFactory() = default; }; From bc1e3e6c518d5f9103e8e02bcbed7c8b434e6ff6 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 24 Jul 2026 12:17:25 -0700 Subject: [PATCH 136/350] Add support for resolving types at plan time. Add an option to prefetch field descriptors when type is known. PiperOrigin-RevId: 953496164 --- conformance/BUILD | 1 + conformance/service.cc | 6 + eval/compiler/BUILD | 1 + eval/compiler/flat_expr_builder.cc | 77 ++++++++++++- eval/eval/BUILD | 10 +- eval/eval/select_step.cc | 118 +++++++++++++++++++- eval/eval/select_step.h | 10 +- eval/eval/select_step_test.cc | 129 ++++++++++++++++++++++ eval/public/cel_options.cc | 50 +++++---- eval/public/cel_options.h | 5 + eval/tests/BUILD | 17 ++- eval/tests/benchmark_test.cc | 85 ++++++++++---- eval/tests/modern_benchmark_test.cc | 104 ++++++++++++----- runtime/internal/BUILD | 1 + runtime/internal/runtime_type_provider.cc | 45 ++------ runtime/internal/runtime_type_provider.h | 5 +- runtime/runtime_options.h | 13 +++ 17 files changed, 554 insertions(+), 123 deletions(-) diff --git a/conformance/BUILD b/conformance/BUILD index 35d554c7b..d72237841 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -64,6 +64,7 @@ cc_library( "//runtime:constant_folding", "//runtime:optional_types", "//runtime:reference_resolver", + "//runtime:regex_precompilation", "//runtime:runtime_options", "//runtime:standard_runtime_builder_factory", "//testutil:test_macros", diff --git a/conformance/service.cc b/conformance/service.cc index d81200cad..4d3815d37 100644 --- a/conformance/service.cc +++ b/conformance/service.cc @@ -76,6 +76,7 @@ #include "runtime/constant_folding.h" #include "runtime/optional_types.h" #include "runtime/reference_resolver.h" +#include "runtime/regex_precompilation.h" #include "runtime/runtime.h" #include "runtime/runtime_options.h" #include "runtime/standard_runtime_builder_factory.h" @@ -283,6 +284,7 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { std::cerr << "Enabling optimizations" << std::endl; options.constant_folding = true; options.constant_arena = constant_arena; + options.enable_typed_field_access = true; } if (select_optimization) { @@ -486,6 +488,9 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { absl::string_view container) { RuntimeOptions options(options_); options.container = std::string(container); + if (enable_optimizations_) { + options.enable_typed_field_access = true; + } CEL_ASSIGN_OR_RETURN( auto builder, CreateStandardRuntimeBuilder( google::protobuf::DescriptorPool::generated_pool(), options)); @@ -493,6 +498,7 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { if (enable_optimizations_) { CEL_RETURN_IF_ERROR(cel::extensions::EnableConstantFolding( builder, google::protobuf::MessageFactory::generated_factory())); + CEL_RETURN_IF_ERROR(cel::extensions::EnableRegexPrecompilation(builder)); } CEL_RETURN_IF_ERROR(cel::EnableReferenceResolver( builder, cel::ReferenceResolverEnabled::kAlways)); diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index ea4dd46b3..2012abeda 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -109,6 +109,7 @@ cc_library( "//common:expr", "//common:kind", "//common:type", + "//common:type_spec_resolver", "//common:value", "//eval/eval:comprehension_step", "//eval/eval:const_value_step", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index d1435b4a3..a650657de 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -30,8 +31,6 @@ #include #include "absl/algorithm/container.h" -#include "absl/base/attributes.h" -#include "absl/base/optimization.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/container/node_hash_map.h" @@ -45,7 +44,6 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "absl/types/variant.h" #include "base/ast.h" @@ -59,6 +57,7 @@ #include "common/expr.h" #include "common/kind.h" #include "common/type.h" +#include "common/type_spec_resolver.h" #include "common/value.h" #include "eval/compiler/check_ast_extensions.h" #include "eval/compiler/flat_expr_builder_extensions.h" @@ -528,7 +527,7 @@ class FlatExprVisitor : public cel::AstVisitor { FlatExprVisitor( const Resolver& resolver, const cel::RuntimeOptions& options, std::vector> program_optimizers, - const absl::flat_hash_map& reference_map, + const absl::flat_hash_map& type_map, const cel::TypeProvider& type_provider, IssueCollector& issue_collector, ProgramBuilder& program_builder, PlannerContext& extension_context, bool enable_optional_types) @@ -538,6 +537,7 @@ class FlatExprVisitor : public cel::AstVisitor { resolved_select_expr_(nullptr), options_(options), program_optimizers_(std::move(program_optimizers)), + type_map_(type_map), issue_collector_(issue_collector), program_builder_(program_builder), extension_context_(extension_context), @@ -606,6 +606,21 @@ class FlatExprVisitor : public cel::AstVisitor { bool PlanRecursiveProgram() const { return max_recursion_depth_ > 0; } + void SetResolvedType(const cel::Expr& expr, cel::Type type) { + resolved_types_[&expr] = std::move(type); + } + + std::optional GetResolvedType(const cel::Expr* expr) const { + if (expr == nullptr) { + return std::nullopt; + } + auto it = resolved_types_.find(expr); + if (it != resolved_types_.end()) { + return it->second; + } + return std::nullopt; + } + void PreVisitExpr(const cel::Expr& expr) override { ValidateOrError(!absl::holds_alternative(expr.kind()), "Invalid empty expression"); @@ -617,6 +632,10 @@ class FlatExprVisitor : public cel::AstVisitor { resume_from_suppressed_branch_ = &expr; } + if (options_.enable_typed_field_access) { + MaybeResolveType(expr); + } + if (block_.has_value()) { BlockInfo& block = *block_; if (block.in && block.bindings_set.contains(&expr)) { @@ -977,6 +996,24 @@ class FlatExprVisitor : public cel::AstVisitor { } StringValue field = cel::StringValue(select_expr.field()); + std::optional struct_type; + std::optional field_type; + if (options_.enable_typed_field_access) { + std::optional operand_type = + GetResolvedType(&select_expr.operand()); + if (operand_type.has_value() && operand_type->IsStruct()) { + struct_type = operand_type->GetStruct(); + if (struct_type.has_value()) { + auto field_lookup = + extension_context_.type_reflector().FindStructTypeFieldByName( + *struct_type, select_expr.field()); + // Swallow error to fallback to duck typing behavior. + if (field_lookup.ok() && field_lookup->has_value()) { + field_type = *std::move(field_lookup); + } + } + } + } if (auto depth = RecursionEligible(); depth.has_value()) { auto deps = ExtractRecursiveDependencies(); if (deps.size() != 1) { @@ -994,6 +1031,13 @@ class FlatExprVisitor : public cel::AstVisitor { return; } + if (field_type.has_value()) { + AddStep(CreateTypedSelectStep( + std::move(field), *struct_type, *std::move(field_type), + select_expr.test_only(), expr.id(), + options_.enable_empty_wrapper_null_unboxing, enable_optional_types_)); + return; + } AddStep(CreateSelectStep( std::move(field), select_expr.test_only(), expr.id(), options_.enable_empty_wrapper_null_unboxing, enable_optional_types_)); @@ -1921,6 +1965,8 @@ class FlatExprVisitor : public cel::AstVisitor { CallHandlerResult HandleHeterogeneousEqualityIn(const cel::Expr& expr, const cel::CallExpr& call); + void MaybeResolveType(const cel::Expr& expr); + const Resolver& resolver_; const cel::TypeProvider& type_provider_; absl::Status progress_status_; @@ -1942,6 +1988,8 @@ class FlatExprVisitor : public cel::AstVisitor { absl::flat_hash_set suppressed_branches_; const cel::Expr* resume_from_suppressed_branch_ = nullptr; std::vector> program_optimizers_; + const absl::flat_hash_map& type_map_; + absl::flat_hash_map resolved_types_; IssueCollector& issue_collector_; ProgramBuilder& program_builder_; @@ -2161,6 +2209,23 @@ FlatExprVisitor::HandleHeterogeneousEqualityIn(const cel::Expr& expr, return CallHandlerResult::kIntercepted; } +void FlatExprVisitor::MaybeResolveType(const cel::Expr& expr) { + // Try to resolve the type from the type map, but don't fail if it's not + // there. This permits cases where the runtime type is compatible but not + // the same as the type checked type. + auto it = type_map_.find(expr.id()); + if (it == type_map_.end()) { + return; + } + absl::StatusOr type = cel::ConvertTypeSpecToType( + it->second, extension_context_.type_reflector(), + extension_context_.MutableArena()); + if (!type.ok()) { + return; + } + SetResolvedType(expr, *type); +} + void LogicalCondVisitor::PreVisit(const cel::Expr* expr) { visitor_->ValidateOrError( !expr->call_expr().has_target() && expr->call_expr().args().size() >= 2, @@ -2561,8 +2626,8 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( // These objects are expected to remain scoped to one build call -- references // to them shouldn't be persisted in any part of the result expression. FlatExprVisitor visitor(resolver, options_, std::move(optimizers), - ast->reference_map(), GetTypeProvider(), - issue_collector, program_builder, extension_context, + ast->type_map(), GetTypeProvider(), issue_collector, + program_builder, extension_context, enable_optional_types_); if (options_.max_recursion_depth == -1 || options_.max_recursion_depth > 0) { diff --git a/eval/eval/BUILD b/eval/eval/BUILD index c0f544405..2597563a4 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -314,13 +314,11 @@ cc_library( ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:expr", + "//common:type", "//common:value", "//common:value_kind", - "//eval/internal:errors", "//internal:status_macros", "//runtime:runtime_options", - "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", @@ -791,7 +789,9 @@ cc_test( deps = [ ":attribute_trail", ":cel_expression_flat_impl", + ":compiler_constant_step", ":const_value_step", + ":create_map_step", ":evaluator_core", ":ident_step", ":select_step", @@ -800,11 +800,14 @@ cc_test( "//common:casting", "//common:expr", "//common:legacy_value", + "//common:type", "//common:value", "//common:value_testing", "//eval/public:activation", "//eval/public:cel_attribute", "//eval/public:cel_value", + "//eval/public:unknown_attribute_set", + "//eval/public:unknown_set", "//eval/public/containers:container_backed_map_impl", "//eval/public/structs:cel_proto_wrapper", "//eval/public/structs:legacy_type_adapter", @@ -831,6 +834,7 @@ cc_test( "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_protobuf//:protobuf", "@com_google_protobuf//:wrappers_cc_proto", ], ) diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 806e20b31..8078f99f8 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -11,6 +11,8 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "common/legacy_value.h" +#include "common/type.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" @@ -182,7 +184,7 @@ class SelectStep : public ExpressionStepBase { absl::Status Evaluate(ExecutionFrame* frame) const override; - private: + protected: cel::StringValue field_value_; std::string field_; bool test_field_presence_; @@ -207,7 +209,7 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { AttributeTrail result_trail; // Handle unknown resolution. - if (frame->enable_unknowns() || frame->enable_missing_attribute_errors()) { + if (frame->attribute_tracking_enabled()) { result_trail = trail.Step(&field_); } @@ -382,6 +384,93 @@ class DirectSelectStep : public DirectExpressionStep { bool enable_optional_types_; }; +class ProtoSelectStep : public SelectStep { + public: + ProtoSelectStep(StringValue value, int64_t expr_id, + bool enable_wrapper_type_null_unboxing, + bool enable_optional_types, + const google::protobuf::Descriptor* descriptor, + const google::protobuf::FieldDescriptor* field_descriptor) + : SelectStep(std::move(value), /*test_field_presence=*/false, expr_id, + enable_wrapper_type_null_unboxing, enable_optional_types), + descriptor_(descriptor), + field_descriptor_(field_descriptor) { + ABSL_DCHECK(descriptor_ != nullptr); + ABSL_DCHECK(field_descriptor_ != nullptr); + } + + absl::Status Evaluate(ExecutionFrame* frame) const override { + if (!frame->value_stack().HasEnough(1)) { + return absl::InternalError( + "No arguments supplied for Select-type expression"); + } + + const Value& arg = frame->value_stack().Peek(); + if (auto unwrapped = arg.AsParsedMessage(); + unwrapped.has_value() && unwrapped->GetDescriptor() == descriptor_) { + return EvaluateModernMessageGetField(frame, *unwrapped); + } else if (const google::protobuf::Message* legacy_message = + cel::interop_internal::GetLegacyMessage(arg); + legacy_message != nullptr && + legacy_message->GetDescriptor() == descriptor_) { + // A little unfortunate, but need to special case for legacy values so we + // can minimize back and forth interop conversions. + return EvaluateLegacyMessageGetField(frame, legacy_message); + } + // If we get an unexpected value type, fall back to the generic + // implementation. + return SelectStep::Evaluate(frame); + } + + private: + absl::Status EvaluateModernMessageGetField( + ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const; + absl::Status EvaluateLegacyMessageGetField( + ExecutionFrame* frame, const google::protobuf::Message* legacy_message) const; + + const google::protobuf::Descriptor* descriptor_; + const google::protobuf::FieldDescriptor* field_descriptor_; +}; + +bool CheckAttributeTrail(const std::string& field, ExecutionFrame* frame) { + if (!frame->attribute_tracking_enabled()) { + return false; + } + AttributeTrail& attr = frame->value_stack().PeekAttribute(); + attr = attr.Step(&field); + + absl::optional marked_attribute_check = + CheckForMarkedAttributes(attr, *frame); + if (marked_attribute_check.has_value()) { + frame->value_stack().Peek() = std::move(marked_attribute_check).value(); + return true; + } + + return false; +} + +absl::Status ProtoSelectStep::EvaluateModernMessageGetField( + ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const { + if (CheckAttributeTrail(field_, frame)) { + return absl::OkStatus(); + } + return parsed_message.GetField( + field_descriptor_, unboxing_option_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), &frame->value_stack().Peek()); +} + +absl::Status ProtoSelectStep::EvaluateLegacyMessageGetField( + ExecutionFrame* frame, const google::protobuf::Message* legacy_message) const { + if (CheckAttributeTrail(field_, frame)) { + return absl::OkStatus(); + } + return cel::interop_internal::WrapLegacyMessageField( + legacy_message, field_descriptor_, unboxing_option_, frame->arena(), + &frame->value_stack().Peek()); +} + } // namespace std::unique_ptr CreateDirectSelectStep( @@ -402,4 +491,29 @@ absl::StatusOr> CreateSelectStep( enable_optional_types); } +// Factory method for Select - based Execution step +absl::StatusOr> CreateTypedSelectStep( + cel::StringValue field, cel::StructType resolved_operand_type, + cel::StructTypeField resolved_field, bool test_only, int64_t expr_id, + bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { + if (!resolved_operand_type.IsMessage() || test_only) { + // The specialization only supports messages. Fallback to the generic + // implementation for other types. + // TODO(uncreated-issue/89): support has() for messages. + return CreateSelectStep(std::move(field), test_only, expr_id, + enable_wrapper_type_null_unboxing, + enable_optional_types); + } + const google::protobuf::Descriptor* descriptor = + resolved_operand_type.GetMessage().descriptor(); + + ABSL_DCHECK(resolved_field.IsMessage()); + const google::protobuf::FieldDescriptor* field_descriptor = + resolved_field.GetMessage().descriptor(); + + return std::make_unique( + std::move(field), expr_id, enable_wrapper_type_null_unboxing, + enable_optional_types, descriptor, field_descriptor); +} + } // namespace google::api::expr::runtime diff --git a/eval/eval/select_step.h b/eval/eval/select_step.h index 2fe35ecb0..c3f965a94 100644 --- a/eval/eval/select_step.h +++ b/eval/eval/select_step.h @@ -5,6 +5,7 @@ #include #include "absl/status/statusor.h" +#include "common/type.h" #include "common/value.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" @@ -17,10 +18,15 @@ std::unique_ptr CreateDirectSelectStep( bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types = false); -// Factory method for Select - based Execution step +// Factory method for Select stack machine based Execution step absl::StatusOr> CreateSelectStep( cel::StringValue field, bool test_only, int64_t expr_id, - bool enable_wrapper_type_null_unboxing, bool enable_optional_types = false); + bool enable_wrapper_type_null_unboxing, bool enable_optional_ytpes = false); + +absl::StatusOr> CreateTypedSelectStep( + cel::StringValue field, cel::StructType resolved_operand_type, + cel::StructTypeField resolved_field, bool test_only, int64_t expr_id, + bool enable_wrapper_type_null_unboxing, bool enable_optional_types); } // namespace google::api::expr::runtime diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index d5ca2aecd..1580472ba 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -19,11 +19,14 @@ #include "common/casting.h" #include "common/expr.h" #include "common/legacy_value.h" +#include "common/type.h" #include "common/value.h" #include "common/value_testing.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" +#include "eval/eval/compiler_constant_step.h" #include "eval/eval/const_value_step.h" +#include "eval/eval/create_map_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/ident_step.h" #include "eval/public/activation.h" @@ -34,6 +37,8 @@ #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/public/testing/matchers.h" +#include "eval/public/unknown_attribute_set.h" +#include "eval/public/unknown_set.h" #include "eval/testutil/test_extensions.pb.h" #include "eval/testutil/test_message.pb.h" #include "extensions/protobuf/value.h" @@ -48,6 +53,7 @@ #include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/descriptor.h" namespace google::api::expr::runtime { @@ -1062,6 +1068,129 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { } } +TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { + ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep("message", -1)); + + cel::StructType resolved_operand_type( + (cel::MessageType(TestAllTypes::descriptor()))); + const google::protobuf::FieldDescriptor* field_desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + ASSERT_NE(field_desc, nullptr); + cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); + + ASSERT_OK_AND_ASSIGN( + auto step1, + CreateTypedSelectStep(cel::StringValue("single_int64"), + resolved_operand_type, resolved_field, + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); + + ExecutionPath path; + path.push_back(std::move(step0)); + path.push_back(std::move(step1)); + cel::RuntimeOptions options; + if (GetParam()) { + options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; + } + CelExpressionFlatImpl cel_expr( + env_, + FlatExpression(std::move(path), /*comprehension_slot_count=*/0, + env_->type_registry.GetComposedTypeProvider(), options)); + + TestAllTypes message; + message.set_single_int64(42); + Activation activation; + activation.InsertValue("message", + CelProtoWrapper::CreateMessage(&message, &arena_)); + + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); + ASSERT_TRUE(result.IsInt64()); + EXPECT_EQ(result.Int64OrDie(), 42); +} + +TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { + ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep("message", -1)); + + cel::StructType resolved_operand_type( + (cel::MessageType(TestAllTypes::descriptor()))); + const google::protobuf::FieldDescriptor* field_desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + ASSERT_NE(field_desc, nullptr); + cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); + + ASSERT_OK_AND_ASSIGN( + auto step1, + CreateTypedSelectStep(cel::StringValue("single_int64"), + resolved_operand_type, resolved_field, + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); + + ExecutionPath path; + path.push_back(std::move(step0)); + path.push_back(std::move(step1)); + cel::RuntimeOptions options; + if (GetParam()) { + options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; + } + CelExpressionFlatImpl cel_expr( + env_, + FlatExpression(std::move(path), /*comprehension_slot_count=*/0, + env_->type_registry.GetComposedTypeProvider(), options)); + + UnknownSet unknown_set(UnknownAttributeSet({CelAttribute("message", {})})); + Activation activation; + activation.InsertValue("message", CelValue::CreateUnknownSet(&unknown_set)); + + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); + ASSERT_TRUE(result.IsUnknownSet()); +} + +TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { + ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep("message", -1)); + + cel::StructType resolved_operand_type( + (cel::MessageType(TestAllTypes::descriptor()))); + const google::protobuf::FieldDescriptor* field_desc = + TestAllTypes::descriptor()->FindFieldByName("single_int64"); + ASSERT_NE(field_desc, nullptr); + cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); + + ASSERT_OK_AND_ASSIGN( + auto step1, + CreateTypedSelectStep(cel::StringValue("single_int64"), + resolved_operand_type, resolved_field, + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); + + ExecutionPath path; + path.push_back(std::move(step0)); + path.push_back(std::move(step1)); + cel::RuntimeOptions options; + options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; + CelExpressionFlatImpl cel_expr( + env_, + FlatExpression(std::move(path), /*comprehension_slot_count=*/0, + env_->type_registry.GetComposedTypeProvider(), options)); + + TestAllTypes message; + message.set_single_int64(42); + + std::vector unknown_patterns; + unknown_patterns.push_back(CelAttributePattern( + "message", {CelAttributeQualifierPattern::OfString("single_int64")})); + + Activation activation; + activation.InsertValue("message", + CelProtoWrapper::CreateMessage(&message, &arena_)); + activation.set_unknown_attribute_patterns(unknown_patterns); + + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); + ASSERT_TRUE(result.IsUnknownSet()); +} + INSTANTIATE_TEST_SUITE_P(UnknownsEnabled, SelectStepConformanceTest, testing::Bool()); diff --git a/eval/public/cel_options.cc b/eval/public/cel_options.cc index 938b5e96f..93b67ad35 100644 --- a/eval/public/cel_options.cc +++ b/eval/public/cel_options.cc @@ -19,29 +19,33 @@ namespace google::api::expr::runtime { cel::RuntimeOptions ConvertToRuntimeOptions(const InterpreterOptions& options) { - return cel::RuntimeOptions{/*.container=*/"", - options.unknown_processing, - options.enable_missing_attribute_errors, - options.enable_timestamp_duration_overflow_errors, - options.short_circuiting, - options.enable_comprehension, - options.comprehension_max_iterations, - options.enable_comprehension_list_append, - options.enable_comprehension_mutable_map, - options.enable_regex, - options.regex_max_program_size, - options.enable_string_conversion, - options.enable_string_concat, - options.enable_list_concat, - options.enable_list_contains, - options.fail_on_warnings, - options.enable_qualified_type_identifiers, - options.enable_heterogeneous_equality, - options.enable_empty_wrapper_null_unboxing, - options.enable_lazy_bind_initialization, - options.max_recursion_depth, - options.enable_recursive_tracing, - options.enable_fast_builtins}; + return cel::RuntimeOptions{ + /*.container=*/"", + options.unknown_processing, + options.enable_missing_attribute_errors, + options.enable_timestamp_duration_overflow_errors, + options.short_circuiting, + options.enable_comprehension, + options.comprehension_max_iterations, + options.enable_comprehension_list_append, + options.enable_comprehension_mutable_map, + options.enable_regex, + options.regex_max_program_size, + options.enable_string_conversion, + options.enable_string_concat, + options.enable_list_concat, + options.enable_list_contains, + options.fail_on_warnings, + options.enable_qualified_type_identifiers, + options.enable_heterogeneous_equality, + options.enable_empty_wrapper_null_unboxing, + options.enable_lazy_bind_initialization, + options.max_recursion_depth, + options.enable_recursive_tracing, + options.enable_fast_builtins, + options.enable_precision_preserving_double_format, + options.enable_typed_field_access, + }; } } // namespace google::api::expr::runtime diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index 4d81eb8a7..001990431 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -218,6 +218,11 @@ struct InterpreterOptions { // Otherwise, will fall back to formatting with the worst-case required // precision. bool enable_precision_preserving_double_format = true; + + // When enabled, the planner will attempt to use a more performant execution + // path for field access when the type is known at plan time, instead of using + // the generic field access implementation. + bool enable_typed_field_access = false; }; // LINT.ThenChange(//depot/google3/runtime/runtime_options.h) diff --git a/eval/tests/BUILD b/eval/tests/BUILD index 9163548d1..c5a7a7062 100644 --- a/eval/tests/BUILD +++ b/eval/tests/BUILD @@ -24,6 +24,12 @@ cc_test( ], deps = [ ":request_context_cc_proto", + "//common:ast_proto", + "//common:decl", + "//common:type", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//eval/public:activation", "//eval/public:builtin_func_registrar", "//eval/public:cel_expr_builder_factory", @@ -42,6 +48,7 @@ cc_test( "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/container:node_hash_set", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", @@ -61,22 +68,25 @@ cc_test( ], deps = [ ":request_context_cc_proto", + "//checker:validation_result", "//common:allocator", "//common:casting", + "//common:decl", "//common:legacy_value", - "//common:memory", "//common:native_type", + "//common:type", "//common:value", + "//compiler", + "//compiler:compiler_factory", + "//compiler:standard_library", "//extensions:comprehensions_v2_functions", "//extensions:comprehensions_v2_macros", "//extensions/protobuf:runtime_adapter", - "//extensions/protobuf:value", "//internal:benchmark", "//internal:testing", "//internal:testing_descriptor_pool", "//internal:testing_message_factory", "//parser", - "//parser:macro", "//parser:macro_registry", "//runtime", "//runtime:activation", @@ -94,7 +104,6 @@ cc_test( "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/eval/tests/benchmark_test.cc b/eval/tests/benchmark_test.cc index f188dc0b7..abeac608a 100644 --- a/eval/tests/benchmark_test.cc +++ b/eval/tests/benchmark_test.cc @@ -12,7 +12,14 @@ #include "absl/container/flat_hash_set.h" #include "absl/container/node_hash_set.h" #include "absl/flags/flag.h" +#include "absl/status/status_matchers.h" #include "absl/strings/match.h" +#include "common/ast_proto.h" +#include "common/decl.h" +#include "common/type.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_expr_builder_factory.h" @@ -31,6 +38,7 @@ ABSL_FLAG(bool, enable_optimizations, false, "enable const folding opt"); ABSL_FLAG(bool, enable_recursive_planning, false, "enable recursive planning"); +ABSL_FLAG(bool, enable_typed_field_access, false, "enable typed field access"); namespace google { namespace api { @@ -39,6 +47,7 @@ namespace runtime { namespace { +using ::absl_testing::IsOk; using ::cel::expr::Expr; using ::cel::expr::ParsedExpr; using ::cel::expr::SourceInfo; @@ -56,6 +65,10 @@ InterpreterOptions GetOptions(google::protobuf::Arena& arena) { options.max_recursion_depth = -1; } + if (absl::GetFlag(FLAGS_enable_typed_field_access)) { + options.enable_typed_field_access = true; + } + return options; } @@ -67,7 +80,8 @@ static void BM_Eval(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); int len = state.range(0); @@ -113,7 +127,8 @@ static void BM_Eval_Trace(benchmark::State& state) { options.enable_recursive_tracing = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); int len = state.range(0); @@ -155,7 +170,8 @@ static void BM_EvalString(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); int len = state.range(0); @@ -198,7 +214,8 @@ static void BM_EvalString_Trace(benchmark::State& state) { options.enable_recursive_tracing = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); int len = state.range(0); @@ -295,7 +312,8 @@ void BM_PolicySymbolic(benchmark::State& state) { options.constant_arena = &arena; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); SourceInfo source_info; ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression( @@ -351,7 +369,8 @@ void BM_PolicySymbolicMap(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); SourceInfo source_info; ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression( @@ -373,22 +392,37 @@ BENCHMARK(BM_PolicySymbolicMap); // Uses a protobuf container for "ip", "path", and "token". void BM_PolicySymbolicProto(benchmark::State& state) { google::protobuf::Arena arena; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( !(request.ip in ["10.0.1.4", "10.0.1.5", "10.0.1.6"]) && ((request.path.startsWith("v1") && request.token in ["v1", "v2", "admin"]) || (request.path.startsWith("v2") && request.token in ["v2", "admin"]) || (request.path.startsWith("/admin") && request.token == "admin" && request.ip in ["10.0.1.1", "10.0.1.2", "10.0.1.3"]) ))cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - SourceInfo source_info; - ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression( - &parsed_expr.expr(), &source_info)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); Activation activation; RequestContext request; @@ -475,7 +509,8 @@ void BM_Comprehension(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); options.comprehension_max_iterations = 10000000; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr, nullptr)); @@ -509,7 +544,8 @@ void BM_Comprehension_Trace(benchmark::State& state) { options.comprehension_max_iterations = 10000000; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr, nullptr)); @@ -531,7 +567,8 @@ void BM_HasMap(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&parsed_expr.expr(), nullptr)); @@ -700,7 +737,8 @@ void BM_ProtoStructAccess(benchmark::State& state) { )cel")); InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&parsed_expr.expr(), nullptr)); @@ -730,7 +768,8 @@ void BM_ProtoListAccess(benchmark::State& state) { )cel")); InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&parsed_expr.expr(), nullptr)); @@ -867,7 +906,8 @@ void BM_NestedComprehension(benchmark::State& state) { InterpreterOptions options = GetOptions(arena); options.comprehension_max_iterations = 10000000; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr, nullptr)); @@ -903,7 +943,8 @@ void BM_NestedComprehension_Trace(benchmark::State& state) { options.enable_recursive_tracing = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr, nullptr)); @@ -936,7 +977,8 @@ void BM_ListComprehension(benchmark::State& state) { options.comprehension_max_iterations = 10000000; options.enable_comprehension_list_append = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN( auto cel_expr, builder->CreateExpression(&(parsed_expr.expr()), nullptr)); @@ -971,7 +1013,8 @@ void BM_ListComprehension_Trace(benchmark::State& state) { options.enable_recursive_tracing = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry(), options)); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); ASSERT_OK_AND_ASSIGN( auto cel_expr, builder->CreateExpression(&(parsed_expr.expr()), nullptr)); @@ -1006,7 +1049,7 @@ void BM_ListComprehension_Opt(benchmark::State& state) { options.comprehension_max_iterations = 10000000; options.enable_comprehension_list_append = true; auto builder = CreateCelExpressionBuilder(options); - ASSERT_OK(RegisterBuiltinFunctions(builder->GetRegistry())); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry()), IsOk()); ASSERT_OK_AND_ASSIGN( auto cel_expr, builder->CreateExpression(&(parsed_expr.expr()), nullptr)); diff --git a/eval/tests/modern_benchmark_test.cc b/eval/tests/modern_benchmark_test.cc index 005f93aa5..28daed981 100644 --- a/eval/tests/modern_benchmark_test.cc +++ b/eval/tests/modern_benchmark_test.cc @@ -14,7 +14,6 @@ // // General benchmarks for CEL evaluator. -#include #include #include #include @@ -36,19 +35,24 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "checker/validation_result.h" #include "common/allocator.h" #include "common/casting.h" +#include "common/decl.h" #include "common/native_type.h" +#include "common/type.h" #include "common/value.h" +#include "compiler/compiler.h" +#include "compiler/compiler_factory.h" +#include "compiler/standard_library.h" #include "eval/tests/request_context.pb.h" #include "extensions/comprehensions_v2_functions.h" #include "extensions/comprehensions_v2_macros.h" #include "extensions/protobuf/runtime_adapter.h" -#include "extensions/protobuf/value.h" #include "internal/benchmark.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" #include "parser/macro_registry.h" #include "parser/parser.h" #include "runtime/activation.h" @@ -62,6 +66,7 @@ #include "google/protobuf/text_format.h" ABSL_FLAG(bool, enable_recursive_planning, false, "enable recursive planning"); +ABSL_FLAG(bool, enable_typed_field_access, false, "enable typed field access"); namespace cel { @@ -85,16 +90,21 @@ RuntimeOptions GetOptions() { options.max_recursion_depth = -1; } + if (absl::GetFlag(FLAGS_enable_typed_field_access)) { + options.enable_typed_field_access = true; + } + return options; } enum class ConstFoldingEnabled { kNo, kYes }; std::unique_ptr StandardRuntimeOrDie( - const cel::RuntimeOptions& options, google::protobuf::Arena* arena = nullptr, + const cel::RuntimeOptions& options, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::Arena* arena = nullptr, ConstFoldingEnabled const_folding = ConstFoldingEnabled::kNo) { - auto builder = CreateStandardRuntimeBuilder( - internal::GetTestingDescriptorPool(), options); + auto builder = CreateStandardRuntimeBuilder(descriptor_pool, options); ABSL_CHECK_OK(builder.status()); switch (const_folding) { @@ -111,13 +121,17 @@ std::unique_ptr StandardRuntimeOrDie( return std::move(runtime).value(); } +std::unique_ptr StandardRuntimeOrDie( + const cel::RuntimeOptions& options, google::protobuf::Arena* arena = nullptr, + ConstFoldingEnabled const_folding = ConstFoldingEnabled::kNo) { + return StandardRuntimeOrDie(options, internal::GetTestingDescriptorPool(), + arena, const_folding); +} + template Value WrapMessageOrDie(const T& message, google::protobuf::Arena* absl_nonnull arena) { - auto value = extensions::ProtoMessageToValue( - message, internal::GetTestingDescriptorPool(), - internal::GetTestingMessageFactory(), arena); - ABSL_CHECK_OK(value.status()); - return std::move(value).value(); + return Value::FromMessage(message, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena); } // Benchmark test @@ -335,7 +349,23 @@ BENCHMARK(BM_PolicyNative); void BM_PolicySymbolic(benchmark::State& state) { google::protobuf::Arena arena; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable( + MakeVariableDecl("ip", StringType())), + IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable( + MakeVariableDecl("path", StringType())), + IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable( + MakeVariableDecl("token", StringType())), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !(ip in ["10.0.1.4", "10.0.1.5", "10.0.1.6"]) && ((path.startsWith("v1") && token in ["v1", "v2", "admin"]) || (path.startsWith("v2") && token in ["v2", "admin"]) || @@ -343,13 +373,14 @@ void BM_PolicySymbolic(benchmark::State& state) { "10.0.1.1", "10.0.1.2", "10.0.1.3" ]) ))cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); RuntimeOptions options = GetOptions(); auto runtime = StandardRuntimeOrDie(options, &arena, ConstFoldingEnabled::kYes); - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); Activation activation; activation.InsertOrAssignValue("ip", StringValue(&arena, kIP)); @@ -437,21 +468,31 @@ class RequestMapImpl : public CustomMapValueInterface { // Uses a lazily constructed map container for "ip", "path", and "token". void BM_PolicySymbolicMap(benchmark::State& state) { google::protobuf::Arena arena; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", + cel::MapType(&arena, cel::StringType(), cel::StringType()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !(request.ip in ["10.0.1.4", "10.0.1.5", "10.0.1.6"]) && ((request.path.startsWith("v1") && request.token in ["v1", "v2", "admin"]) || (request.path.startsWith("v2") && request.token in ["v2", "admin"]) || (request.path.startsWith("/admin") && request.token == "admin" && request.ip in ["10.0.1.1", "10.0.1.2", "10.0.1.3"]) ))cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - SourceInfo source_info; - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); Activation activation; CustomMapValue map_value(google::protobuf::Arena::Create(&arena), @@ -472,21 +513,31 @@ BENCHMARK(BM_PolicySymbolicMap); // Uses a protobuf container for "ip", "path", and "token". void BM_PolicySymbolicProto(benchmark::State& state) { google::protobuf::Arena arena; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !(request.ip in ["10.0.1.4", "10.0.1.5", "10.0.1.6"]) && ((request.path.startsWith("v1") && request.token in ["v1", "v2", "admin"]) || (request.path.startsWith("v2") && request.token in ["v2", "admin"]) || (request.path.startsWith("/admin") && request.token == "admin" && request.ip in ["10.0.1.1", "10.0.1.2", "10.0.1.3"]) ))cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); - auto runtime = StandardRuntimeOrDie(options); - - SourceInfo source_info; - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); Activation activation; RequestContext request; @@ -497,8 +548,7 @@ void BM_PolicySymbolicProto(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.IsBool() && result.GetBool().NativeValue()); } } diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index f1141ef08..6c96de1cf 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -194,6 +194,7 @@ cc_library( srcs = ["runtime_type_provider.cc"], hdrs = ["runtime_type_provider.h"], deps = [ + "//common:descriptor_pool_type_introspector", "//common:type", "//common:value", "//internal:status_macros", diff --git a/runtime/internal/runtime_type_provider.cc b/runtime/internal/runtime_type_provider.cc index e6fd55d2c..5314918bf 100644 --- a/runtime/internal/runtime_type_provider.cc +++ b/runtime/internal/runtime_type_provider.cc @@ -14,6 +14,7 @@ #include "runtime/internal/runtime_type_provider.h" +#include #include #include "absl/base/nullability.h" @@ -28,7 +29,6 @@ #include "common/value.h" #include "common/values/value_builder.h" #include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace cel::runtime_internal { @@ -48,9 +48,14 @@ absl::StatusOr> RuntimeTypeProvider::FindTypeImpl( if (type.has_value()) { return type; } - const auto* desc = descriptor_pool_->FindMessageTypeByName(name); - if (desc != nullptr) { - return MessageType(desc); + + auto result = descriptor_pool_provider_.FindType(name); + if (!result.ok()) { + return result; + } + + if (result->has_value() && !result->value().IsEnum()) { + return result; } if (const auto it = types_.find(name); it != types_.end()) { @@ -66,24 +71,7 @@ RuntimeTypeProvider::FindEnumConstantImpl(absl::string_view type, if (enum_constant.has_value()) { return enum_constant; } - const google::protobuf::EnumDescriptor* enum_desc = - descriptor_pool_->FindEnumTypeByName(type); - if (enum_desc == nullptr) { - return std::nullopt; - } - - // Note: we don't support strong enum typing at this time so only the fully - // qualified enum values are meaningful, so we don't provide any signal if the - // enum type is found but can't match the value name. - const google::protobuf::EnumValueDescriptor* value_desc = - enum_desc->FindValueByName(value); - if (value_desc == nullptr) { - return std::nullopt; - } - - return TypeIntrospector::EnumConstant{ - EnumType(enum_desc), enum_desc->full_name(), value_desc->name(), - value_desc->number()}; + return descriptor_pool_provider_.FindEnumConstant(type, value); } absl::StatusOr> @@ -93,18 +81,7 @@ RuntimeTypeProvider::FindStructTypeFieldByNameImpl( if (field.has_value()) { return field; } - const auto* desc = descriptor_pool_->FindMessageTypeByName(type); - if (desc == nullptr) { - return std::nullopt; - } - const auto* field_desc = desc->FindFieldByName(name); - if (field_desc == nullptr) { - field_desc = descriptor_pool_->FindExtensionByPrintableName(desc, name); - if (field_desc == nullptr) { - return std::nullopt; - } - } - return MessageTypeField(field_desc); + return descriptor_pool_provider_.FindStructTypeFieldByName(type, name); } absl::StatusOr diff --git a/runtime/internal/runtime_type_provider.h b/runtime/internal/runtime_type_provider.h index 3f418af4d..fc7163f6c 100644 --- a/runtime/internal/runtime_type_provider.h +++ b/runtime/internal/runtime_type_provider.h @@ -21,6 +21,7 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "common/descriptor_pool_type_introspector.h" #include "common/type.h" #include "common/type_reflector.h" #include "common/value.h" @@ -34,7 +35,8 @@ class RuntimeTypeProvider final : public TypeReflector { public: explicit RuntimeTypeProvider( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool) - : descriptor_pool_(descriptor_pool) {} + : descriptor_pool_(descriptor_pool), + descriptor_pool_provider_(descriptor_pool) {} absl::Status RegisterType(const OpaqueType& type); @@ -55,6 +57,7 @@ class RuntimeTypeProvider final : public TypeReflector { private: const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; + DescriptorPoolTypeIntrospector descriptor_pool_provider_; absl::flat_hash_map types_; }; diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 7a61208a0..072daf26c 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -188,6 +188,19 @@ struct RuntimeOptions { // // If disabled, will use the legacy behavior of rounding to 6 decimal places. bool enable_precision_preserving_double_format = true; + + // When enabled, the planner will attempt to use a more performant execution + // path for field access when the type is known at plan time, instead of using + // the generic field access implementation. + // + // The runtime will try to verify that the field access is compatible with the + // actual type at evaluation time, and will fall back to the generic + // implementation if the value is not what was expected. + // + // This is not recommended if the values bound to the activation are typically + // not what the planner expected (e.g. a map that was declared as a proto or + // a different message with matching field names). + bool enable_typed_field_access = false; }; // LINT.ThenChange(//depot/google3/eval/public/cel_options.h) From 62788abcb369b9646aba24da85a2b4c79ab38042 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 24 Jul 2026 12:53:12 -0700 Subject: [PATCH 137/350] [Pratt Parser] Add support for options.error_recovery_limit PiperOrigin-RevId: 953512384 --- parser/internal/pratt_parser.cc | 6 +- parser/internal/pratt_parser_test.cc | 121 +++++++++++++++---------- parser/internal/pratt_parser_worker.cc | 32 ++++++- parser/internal/pratt_parser_worker.h | 6 +- parser/parser_test.cc | 2 +- 5 files changed, 109 insertions(+), 58 deletions(-) diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc index c321fa7b8..e83c21fce 100644 --- a/parser/internal/pratt_parser.cc +++ b/parser/internal/pratt_parser.cc @@ -50,9 +50,11 @@ namespace { std::string DisplayParserError(const cel::Source& source, SourceLocation location, std::string_view message) { + int32_t display_column = + location.column >= 0 ? location.column + 1 : location.column; return absl::StrCat( - absl::StrFormat("ERROR: %s:%zu:%zu: %s", source.description(), - location.line, location.column + 1, message), + absl::StrFormat("ERROR: %s:%d:%d: %s", source.description(), + location.line, display_column, message), source.DisplayErrorLocation(location)); } diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 708936a4e..5dd3bb182 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -1055,7 +1055,7 @@ std::string FormatIssues(const cel::Source& source, issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { absl::StrAppend( out, - absl::StrFormat("ERROR: %s:%zu:%zu: %s", source.description(), + absl::StrFormat("ERROR: %s:%d:%d: %s", source.description(), issue.location().line, issue.location().column + 1, issue.message()), source.DisplayErrorLocation(issue.location())); @@ -1576,53 +1576,6 @@ INSTANTIATE_TEST_SUITE_P(PrattParserMacroTest, PrattParserMacroTest, testing::ValuesIn(GetMacroTestCases()), TestName); -TEST(PrattParserMacroErrorTest, ReportError) { - auto builder = NewPrattParserBuilder(); - ASSERT_OK_AND_ASSIGN( - auto error_macro, - Macro::Global("bad_macro", 1, - [](MacroExprFactory& macro_factory, - absl::Span args) -> std::optional { - return macro_factory.ReportError("custom macro error"); - })); - - ASSERT_THAT(builder->AddMacro(error_macro), IsOk()); - ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); - - ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("42 + bad_macro(x)")); - std::vector issues; - auto ast = parser->Parse(*source, &issues); - EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_EQ(FormatIssues(*source, issues), - "ERROR: :1:6: custom macro error\n" - " | 42 + bad_macro(x)\n" - " | .....^"); -} - -TEST(PrattParserMacroErrorTest, ReportErrorAt) { - auto builder = NewPrattParserBuilder(); - ASSERT_OK_AND_ASSIGN( - auto error_at_macro, - Macro::Global("bad_macro_at", 1, - [](MacroExprFactory& macro_factory, - absl::Span args) -> std::optional { - return macro_factory.ReportErrorAt(args[0], - "custom error at arg"); - })); - - ASSERT_THAT(builder->AddMacro(error_at_macro), IsOk()); - ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); - - ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("bad_macro_at(x)")); - std::vector issues; - auto ast = parser->Parse(*source, &issues); - EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_EQ(FormatIssues(*source, issues), - "ERROR: :1:14: custom error at arg\n" - " | bad_macro_at(x)\n" - " | .............^"); -} - TEST(PrattParserMacroCallsTest, MacroCallsDisabledByDefault) { cel::ParserOptions options; options.add_macro_calls = false; @@ -1737,5 +1690,77 @@ TEST(PrattParserMacroCallsTest, NestedMacroCallsUseCopyAndReplaceReplacer) { )")); } +TEST(PrattParserMacroErrorTest, ReportError) { + auto builder = NewPrattParserBuilder(); + ASSERT_OK_AND_ASSIGN( + auto error_macro, + Macro::Global("bad_macro", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.ReportError("custom macro error"); + })); + + ASSERT_THAT(builder->AddMacro(error_macro), IsOk()); + ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("42 + bad_macro(x)")); + std::vector issues; + auto ast = parser->Parse(*source, &issues); + EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :1:6: custom macro error\n" + " | 42 + bad_macro(x)\n" + " | .....^"); +} + +TEST(PrattParserMacroErrorTest, ReportErrorAt) { + auto builder = NewPrattParserBuilder(); + ASSERT_OK_AND_ASSIGN( + auto error_at_macro, + Macro::Global("bad_macro_at", 1, + [](MacroExprFactory& macro_factory, + absl::Span args) -> std::optional { + return macro_factory.ReportErrorAt(args[0], + "custom error at arg"); + })); + + ASSERT_THAT(builder->AddMacro(error_at_macro), IsOk()); + ASSERT_OK_AND_ASSIGN(auto parser, builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("bad_macro_at(x)")); + std::vector issues; + auto ast = parser->Parse(*source, &issues); + EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :1:14: custom error at arg\n" + " | bad_macro_at(x)\n" + " | .............^"); +} + +TEST(PrattParserErrorRecoveryTest, ErrorRecoveryLimitZero) { + cel::ParserOptions options; + options.error_recovery_limit = 0; + std::vector issues; + auto result = Parse("......", options, &issues); + EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument)); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("......")); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :-1:0: Error recovery limit (0) exceeded"); +} + +TEST(PrattParserErrorRecoveryTest, ErrorRecoveryLimitOne) { + cel::ParserOptions options; + options.error_recovery_limit = 1; + std::vector issues; + auto result = Parse("......", options, &issues); + EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument)); + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("......")); + EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :1:2: expected identifier\n" + " | ......\n" + " | .^\n" + "ERROR: :-1:0: Error recovery limit (1) exceeded"); +} + } // namespace } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index 4e40adc1e..8dece29a8 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -20,6 +20,7 @@ #include "absl/base/nullability.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "common/source.h" #include "parser/internal/lexer.h" @@ -50,6 +51,9 @@ std::string ParserWorker::GetTokenText(const Token& tok) const { } Token ParserWorker::NextSignificantToken() { + if (is_recovery_limit_exceeded()) { + return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + } while (true) { Token tok = lexer_.Lex(); if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) { @@ -57,6 +61,9 @@ Token ParserWorker::NextSignificantToken() { } if (tok.type == TokenType::kError) { ReportError(tok, lexer_.GetError().message); + if (is_recovery_limit_exceeded()) { + return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + } } return tok; } @@ -64,6 +71,10 @@ Token ParserWorker::NextSignificantToken() { Token ParserWorker::NextToken() { current_token_ = peek_token_; + if (is_recovery_limit_exceeded()) { + peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + return current_token_; + } if (peek_token_.type != TokenType::kEnd) { peek_token_ = NextSignificantToken(); } @@ -75,6 +86,9 @@ bool ParserWorker::Expect(TokenType type, absl::string_view msg) { NextToken(); return true; } + if (is_recovery_limit_exceeded()) { + return false; + } if (peek_token_.type != TokenType::kError) { std::string err_msg; if (msg.empty()) { @@ -98,9 +112,7 @@ bool ParserWorker::Expect(TokenType type, absl::string_view msg) { void ParserWorker::SynchronizeOnDelimiter() { if (is_recovery_limit_exceeded()) { - while (peek_token_.type != TokenType::kEnd) { - NextToken(); - } + peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; return; } while (peek_token_.type != TokenType::kEnd) { @@ -149,8 +161,20 @@ void ParserWorker::ReportError(int32_t position, absl::string_view msg) { void ParserWorker::ReportError(const SourceLocation& loc, absl::string_view msg) { + if (error_count_ > options_.error_recovery_limit) { + return; + } error_count_++; - if (parse_issues_ != nullptr) { + if (error_count_ == options_.error_recovery_limit + 1) { + if (parse_issues_ != nullptr) { + parse_issues_->push_back( + cel::ParseIssue(absl::StrFormat("Error recovery limit (%d) exceeded", + options_.error_recovery_limit))); + } + peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + } + if (parse_issues_ != nullptr && + error_count_ <= options_.error_recovery_limit) { parse_issues_->push_back(cel::ParseIssue(loc, std::string(msg))); } } diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 45221714f..a7dee687c 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -81,7 +81,7 @@ class ParserWorker { // Error reporting and recovery bool is_recovery_limit_exceeded() const { - return error_count_ >= options_.error_recovery_limit; + return error_count_ > options_.error_recovery_limit; } void ReportError(int32_t position, absl::string_view msg); void ReportError(const SourceLocation& loc, absl::string_view msg); @@ -214,7 +214,7 @@ class PrattParserWorker : public ParserWorker { template ExprNode PrattParserWorker::Parse() { ExprNode expr = ParseExpr(); - if (is_recursion_limit_exceeded()) { + if (is_recursion_limit_exceeded() || is_recovery_limit_exceeded()) { return expr; } if (peek_token_.type != TokenType::kEnd && @@ -226,7 +226,7 @@ ExprNode PrattParserWorker::Parse() { template ExprNode PrattParserWorker::ParseExpr() { - if (recursion_limit_exceeded_) { + if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { return ExprNode(); } if (recursion_depth_ > options_.max_recursion_depth) { diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 35f11b413..357a83a79 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1584,7 +1584,7 @@ TEST(ExpressionTest, TsanOom) { .IgnoreError(); } -TEST(ExpressionTest, ErrorRecoveryLimits) { +TEST_P(ExpressionTest, ErrorRecoveryLimits) { ParserOptions options; options.error_recovery_limit = 1; auto result = Parse("......", "", options); From a0f5a4fcfdc529876bf389e5ca27cc5b200f2c4e Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 24 Jul 2026 13:02:58 -0700 Subject: [PATCH 138/350] [Pratt Parser] Call-Stack Optimization PiperOrigin-RevId: 953516565 --- parser/internal/BUILD | 1 - parser/internal/pratt_parser_test.cc | 6 + parser/internal/pratt_parser_worker.cc | 67 +++++ parser/internal/pratt_parser_worker.h | 377 +++++++++++++------------ 4 files changed, 262 insertions(+), 189 deletions(-) diff --git a/parser/internal/BUILD b/parser/internal/BUILD index e5593a9e2..e3f2b8ed0 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -92,7 +92,6 @@ cc_library( "//parser:options", "//parser:parser_interface", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 5dd3bb182..26f849639 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -223,6 +223,12 @@ std::vector GetParserTestCases() { -123^#1:int64# )", }, + TestCase{ + .source = "9223372036854775807", + .expected_ast = R"( + 9223372036854775807^#1:int64# + )", + }, TestCase{ .source = "-9223372036854775808", .expected_ast = R"( diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index 8dece29a8..c502796f5 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -22,12 +22,79 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" +#include "common/operators.h" #include "common/source.h" #include "parser/internal/lexer.h" #include "parser/options.h" #include "parser/parser_interface.h" namespace cel::parser_internal { +namespace { + +using CelOperator = ::google::api::expr::common::CelOperator; + +const BinaryOpInfo kLogicalOr = {1, CelOperator::LOGICAL_OR, true, + TokenType::kLogicalOr}; +const BinaryOpInfo kLogicalAnd = {2, CelOperator::LOGICAL_AND, true, + TokenType::kLogicalAnd}; +const BinaryOpInfo kLess = {3, CelOperator::LESS, false, TokenType::kLess}; +const BinaryOpInfo kLessEqual = {3, CelOperator::LESS_EQUALS, false, + TokenType::kLessEqual}; +const BinaryOpInfo kGreater = {3, CelOperator::GREATER, false, + TokenType::kGreater}; +const BinaryOpInfo kGreaterEqual = {3, CelOperator::GREATER_EQUALS, false, + TokenType::kGreaterEqual}; +const BinaryOpInfo kEqualEqual = {3, CelOperator::EQUALS, false, + TokenType::kEqualEqual}; +const BinaryOpInfo kExclamationEqual = {3, CelOperator::NOT_EQUALS, false, + TokenType::kExclamationEqual}; +const BinaryOpInfo kIn = {3, CelOperator::IN, false, TokenType::kIn}; +const BinaryOpInfo kPlus = {4, CelOperator::ADD, false, TokenType::kPlus}; +const BinaryOpInfo kMinus = {4, CelOperator::SUBTRACT, false, + TokenType::kMinus}; +const BinaryOpInfo kAsterisk = {5, CelOperator::MULTIPLY, false, + TokenType::kAsterisk}; +const BinaryOpInfo kSlash = {5, CelOperator::DIVIDE, false, TokenType::kSlash}; +const BinaryOpInfo kPercent = {5, CelOperator::MODULO, false, + TokenType::kPercent}; +const BinaryOpInfo kDefaultOpInfo = {0, "", false, TokenType::kError}; + +} // namespace + +const BinaryOpInfo& GetBinaryOpInfo(TokenType type) { + switch (type) { + case TokenType::kLogicalOr: + return kLogicalOr; + case TokenType::kLogicalAnd: + return kLogicalAnd; + case TokenType::kLess: + return kLess; + case TokenType::kLessEqual: + return kLessEqual; + case TokenType::kGreater: + return kGreater; + case TokenType::kGreaterEqual: + return kGreaterEqual; + case TokenType::kEqualEqual: + return kEqualEqual; + case TokenType::kExclamationEqual: + return kExclamationEqual; + case TokenType::kIn: + return kIn; + case TokenType::kPlus: + return kPlus; + case TokenType::kMinus: + return kMinus; + case TokenType::kAsterisk: + return kAsterisk; + case TokenType::kSlash: + return kSlash; + case TokenType::kPercent: + return kPercent; + default: + return kDefaultOpInfo; + } +} ParserWorker::ParserWorker( const cel::Source& source, const cel::ParserOptions& options, diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index a7dee687c..bca1de6b1 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -25,7 +25,6 @@ #include #include "absl/base/nullability.h" -#include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_map.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" @@ -103,6 +102,13 @@ class ParserWorker { bool recursion_limit_exceeded_ = false; }; +struct BinaryOpInfo { + int precedence = 0; + absl::string_view name; + bool is_logical = false; + TokenType type = TokenType::kError; +}; + // Generic Pratt parser implementation parameterized by the AST node type // (`ExprNode`). // @@ -173,13 +179,72 @@ class PrattParserWorker : public ParserWorker { using CelOperator = ::google::api::expr::common::CelOperator; ExprNode ParseExpr(); - ExprNode ParseConditionalOr(); - ExprNode ParseConditionalAnd(); - ExprNode ParseRelation(); - ExprNode ParseCalc(int min_prec); + + // Parses binary operator expressions and ternary conditional expressions + // (`? :`) using operator-precedence (Pratt) parsing. Consumes operators from + // the token stream whose binding precedence is greater than or equal to + // `min_prec`. + // + // Example (`min_prec = 0` on `a + b * c`): Parses prefix/postfix expression + // `a`, encounters `+` (precedence 1 >= 0), recurses with `ParseBinary(2)` to + // parse `b * c` at higher precedence, and builds the `+` call node. + // + // Example (`min_prec = 0` on `a ? b : c`): Parses condition `a`, encounters + // `?`, consumes `?`, and recurses with `ParseBinary(1)` for true branch `b` + // and `ParseBinary(0)` for false branch `c`. + ExprNode ParseBinaryAndTernary(int min_prec); + + // Helper method for parsing a contiguous chain of same-precedence logical + // operators (`&&` or `||`) iteratively into a list of terms and operator IDs. + // Constructs either a balanced binary AST (`(a && b) && c`) or a single + // variadic call (`_&&_(a, b, c)`) via `BalanceLogical`. This avoids deep C++ + // recursion when processing expressions with dozens of chained logical terms. + // + // Example (`a && b && c && d`): Iteratively collects terms `[a, b, c, d]` and + // builds `((a && b) && c) && d` without ascending/descending C++ stack + // frames for each term. + ExprNode ParseBalancedLogicalChain(ExprNode lhs, const BinaryOpInfo& op_info); + + // Parses prefix unary operators (`!`, `-`) and trailing postfix + // member/indexing operations (`.field`, `[index]`, `.method(args)`). First + // calls `ParseUnary()` to obtain the base operand, then consumes trailing + // selector + // (`.`) and bracket (`[` or `{`) operations in an iterative loop. + // + // Example (`!a.b[0].c(x)`): `ParseUnary()` consumes `!` and calls + // `ParseSelectorChain()` for `a.b[0].c(x)`, which parses primary `a` and + // then loops iteratively through `.b`, `[0]`, and `.c(x)`. + ExprNode ParseSelectorChain(); + + // Parses prefix unary operators (logical NOT `!` and negation `-`). If a + // numeric literal immediately follows `-`, folds it directly into a negative + // constant node (`-42`, `-3.14`). Otherwise, wraps the operand in a `_!_` or + // `_-_` function call node. Delegates to `ParseSelectorChain()` for the + // operand to naturally support chained prefix operations (`!!x`, `-!y`), or + // falls back to `ParsePrimary()` if no prefix operator is encountered. + // + // Example (`-42`): Folds directly into a negative integer constant `-42`. + // Example (`!has(x.y)`): Consumes `!` and creates a `LOGICAL_NOT` call node + // wrapping `has(x.y)`. ExprNode ParseUnary(); - ExprNode ParseMember(); + + // Parses primary leaf expressions (`nud` atomic atoms), including + // parenthesized expressions (`(expr)`), literal constants (`null`, `true`, + // `false`, numbers, strings, bytes), collection initializers (`[list]`, + // `{map}`), identifiers + // (`foo`), and global or namespace function/macro calls (`foo(args)`, + // `.pkg.func(args)`). + // + // For parenthesized expressions and collection literals, delegates back to + // `ParseExpr()` or collection subroutines (`ParseList()`, `ParseMap()`). + // For global function calls (`foo(args)`), attempts macro expansion before + // falling back to standard call node construction. + // + // Example (`(a + b)`): Consumes `(`, recurses to `ParseExpr()`, and expects + // `)`. Example (`has(x.y)`): Consumes `has`, parses arguments `(x.y)`, and + // expands the `has` macro. ExprNode ParsePrimary(); + ExprNode ParseList(); ExprNode ParseMap(); ExprNode ParseStruct(int64_t obj_id, absl::string_view struct_name); @@ -234,209 +299,77 @@ ExprNode PrattParserWorker::ParseExpr() { return ExprNode(); } recursion_depth_++; - auto recursion_cleanup = absl::MakeCleanup([this]() { recursion_depth_--; }); - ExprNode lhs = ParseConditionalOr(); - if (peek_token_.type == TokenType::kQuestion) { - NextToken(); - int64_t op_id = NextId(); - ExprNode true_expr = ParseConditionalOr(); - if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { - return lhs; - } - ExprNode false_expr = ParseExpr(); - return ast_factory_.NewCall( - op_id, CelOperator::CONDITIONAL, - std::vector{std::move(lhs), std::move(true_expr), - std::move(false_expr)}); - } - return lhs; -} - -// Parses logical OR expressions (e.g., `a || b || c`). -template -ExprNode PrattParserWorker::ParseConditionalOr() { - ExprNode lhs = ParseConditionalAnd(); - if (peek_token_.type != TokenType::kLogicalOr) { - return lhs; - } - std::vector terms; - std::vector ops; - terms.push_back(std::move(lhs)); - while (peek_token_.type == TokenType::kLogicalOr) { - Token op_tok = NextToken(); - ExprNode rhs = ParseConditionalAnd(); - ops.push_back(NextId(op_tok)); - terms.push_back(std::move(rhs)); - } - return BalanceLogical(CelOperator::LOGICAL_OR, std::move(terms), - std::move(ops), - options_.enable_variadic_logical_operators); + ExprNode expr = ParseBinaryAndTernary(0); + recursion_depth_--; + return expr; } -// Parses logical AND expressions (e.g., `a && b && c`). -template -ExprNode PrattParserWorker::ParseConditionalAnd() { - ExprNode lhs = ParseRelation(); - if (peek_token_.type != TokenType::kLogicalAnd) { - return lhs; - } - std::vector terms; - std::vector ops; - terms.push_back(std::move(lhs)); - while (peek_token_.type == TokenType::kLogicalAnd) { - Token op_tok = NextToken(); - ExprNode rhs = ParseRelation(); - ops.push_back(NextId(op_tok)); - terms.push_back(std::move(rhs)); - } - return BalanceLogical(CelOperator::LOGICAL_AND, std::move(terms), - std::move(ops), - options_.enable_variadic_logical_operators); -} +const BinaryOpInfo& GetBinaryOpInfo(TokenType type); -// Parses relational & equality ops (e.g., `a < b`, `x == y`, `a in b`). +// Parses binary operator expressions and ternary conditional expressions +// (`? :`) using Pratt operator-precedence parsing (e.g., `a + b * c`). template -ExprNode PrattParserWorker::ParseRelation() { - ExprNode lhs = ParseCalc(0); +ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { + ExprNode lhs = ParseSelectorChain(); while (true) { TokenType tok = peek_token_.type; - absl::string_view op_name; - switch (tok) { - case TokenType::kLess: - op_name = CelOperator::LESS; - break; - case TokenType::kLessEqual: - op_name = CelOperator::LESS_EQUALS; - break; - case TokenType::kGreater: - op_name = CelOperator::GREATER; - break; - case TokenType::kGreaterEqual: - op_name = CelOperator::GREATER_EQUALS; - break; - case TokenType::kEqualEqual: - op_name = CelOperator::EQUALS; - break; - case TokenType::kExclamationEqual: - op_name = CelOperator::NOT_EQUALS; - break; - case TokenType::kIn: - op_name = CelOperator::IN; - break; - default: + if (tok == TokenType::kQuestion && min_prec <= 0) { + NextToken(); + int64_t op_id = NextId(); + ExprNode true_expr = ParseBinaryAndTernary(1); + if (!Expect(TokenType::kColon, + "expected ':' in conditional expression")) { return lhs; + } + ExprNode false_expr = ParseBinaryAndTernary(0); + lhs = ast_factory_.NewCall( + op_id, CelOperator::CONDITIONAL, + std::vector{std::move(lhs), std::move(true_expr), + std::move(false_expr)}); + continue; } - Token op = NextToken(); - int64_t op_id = NextId(op); - ExprNode rhs = ParseCalc(0); - lhs = ast_factory_.NewCall( - op_id, std::string(op_name), - std::vector{std::move(lhs), std::move(rhs)}); - } - return lhs; -} -// Parses arithmetic calculation expressions (e.g., `a + b * c`, `x - y % z`). -template -ExprNode PrattParserWorker::ParseCalc(int min_prec) { - ExprNode lhs = ParseUnary(); - while (true) { - TokenType tok = peek_token_.type; - int prec = 0; - absl::string_view op_name; - if (tok == TokenType::kAsterisk) { - prec = 2; - op_name = CelOperator::MULTIPLY; - } else if (tok == TokenType::kSlash) { - prec = 2; - op_name = CelOperator::DIVIDE; - } else if (tok == TokenType::kPercent) { - prec = 2; - op_name = CelOperator::MODULO; - } else if (tok == TokenType::kPlus) { - prec = 1; - op_name = CelOperator::ADD; - } else if (tok == TokenType::kMinus) { - prec = 1; - op_name = CelOperator::SUBTRACT; - } else { - break; + const BinaryOpInfo& op_info = GetBinaryOpInfo(tok); + if (op_info.precedence < min_prec || op_info.precedence == 0) break; + + if (op_info.is_logical) { + lhs = ParseBalancedLogicalChain(std::move(lhs), op_info); + continue; } - if (prec < min_prec) break; - Token op = NextToken(); - int64_t op_id = NextId(op); - ExprNode rhs = ParseCalc(prec + 1); + Token op_tok = NextToken(); + int64_t op_id = NextId(op_tok); + ExprNode rhs = ParseBinaryAndTernary(op_info.precedence + 1); lhs = ast_factory_.NewCall( - op_id, std::string(op_name), + op_id, std::string(op_info.name), std::vector{std::move(lhs), std::move(rhs)}); } return lhs; } -// Parses unary logical NOT and negation expressions (e.g., `!a`, `-b`). +// Parses continuous chains of logical operators (`&&`, `||`) iteratively +// (e.g., `a && b && c`) and constructs a balanced or variadic AST. template -ExprNode PrattParserWorker::ParseUnary() { - TokenType tok = peek_token_.type; - if (tok == TokenType::kExclamation) { - Token op = NextToken(); - int64_t op_id = NextId(op); - ExprNode operand = ParseUnary(); - return ast_factory_.NewCall(op_id, std::string(CelOperator::LOGICAL_NOT), - std::vector{std::move(operand)}); - } - if (tok == TokenType::kMinus) { - Token op = NextToken(); - if (peek_token_.type == TokenType::kInt) { - Token lit_tok = NextToken(); - std::string text = GetTokenText(lit_tok); - int64_t int_val = 0; - bool success = false; - if (absl::StartsWith(text, "0x") || absl::StartsWith(text, "0X")) { - uint64_t uint_val = 0; - if (absl::SimpleHexAtoi(text, &uint_val)) { - if (uint_val <= uint64_t{0x8000000000000000}) { - if (uint_val == uint64_t{0x8000000000000000}) { - int_val = std::numeric_limits::min(); - } else { - int_val = -static_cast(uint_val); - } - success = true; - } - } - } else { - std::string val = absl::StrCat("-", text); - success = absl::SimpleAtoi(val, &int_val); - } - if (success) { - return ast_factory_.NewIntConst(NextId(op.start), int_val); - } - ReportError(lit_tok, "invalid int literal"); - return ast_factory_.NewUnspecified(NextId(lit_tok)); - } - if (peek_token_.type == TokenType::kFloat) { - Token lit_tok = NextToken(); - std::string val = absl::StrCat("-", GetTokenText(lit_tok)); - double double_val = 0.0; - if (absl::SimpleAtod(val, &double_val)) { - return ast_factory_.NewDoubleConst(NextId(op.start), double_val); - } - ReportError(lit_tok, "invalid double literal"); - return ast_factory_.NewUnspecified(NextId(lit_tok)); - } - // Regular negate call - int64_t op_id = NextId(op); - ExprNode operand = ParseUnary(); - return ast_factory_.NewCall(op_id, std::string(CelOperator::NEGATE), - std::vector{std::move(operand)}); +ExprNode PrattParserWorker::ParseBalancedLogicalChain( + ExprNode lhs, const BinaryOpInfo& op_info) { + std::vector terms; + std::vector ops; + terms.push_back(std::move(lhs)); + while (peek_token_.type == op_info.type) { + Token op_tok = NextToken(); + ExprNode rhs = ParseBinaryAndTernary(op_info.precedence + 1); + ops.push_back(NextId(op_tok)); + terms.push_back(std::move(rhs)); } - return ParseMember(); + return BalanceLogical(op_info.name, std::move(terms), std::move(ops), + options_.enable_variadic_logical_operators); } -// Parses member calls & indexing (e.g., `a.b`, `a.b(c)`, `a[b]`, `a.?b`). +// Parses prefix and postfix member/indexing operations iteratively +// (e.g., `!a.b[0].c(x)`). template -ExprNode PrattParserWorker::ParseMember() { - ExprNode lhs = ParsePrimary(); +ExprNode PrattParserWorker::ParseSelectorChain() { + ExprNode lhs = ParseUnary(); while (true) { TokenType tok = peek_token_.type; if (tok == TokenType::kDot) { @@ -513,7 +446,75 @@ ExprNode PrattParserWorker::ParseMember() { return lhs; } -// Parses primary expressions & literals (e.g., `null`, `true`, `x`, +// Parses prefix unary operators (`!`, `-`) and folds negative numeric literals +// (e.g., `-42`, `!has(x.y)`). +template +ExprNode PrattParserWorker::ParseUnary() { + TokenType tok = peek_token_.type; + if (tok == TokenType::kExclamation) { + Token op = NextToken(); + int64_t op_id = NextId(op); + ExprNode operand = ParseSelectorChain(); + return ast_factory_.NewCall(op_id, std::string(CelOperator::LOGICAL_NOT), + std::vector{std::move(operand)}); + } + if (tok == TokenType::kMinus) { + Token op = NextToken(); + if (peek_token_.type == TokenType::kInt) { + Token lit_tok = NextToken(); + std::string text = GetTokenText(lit_tok); + int64_t int_val = 0; + bool success = false; + if (absl::StartsWith(text, "0x") || absl::StartsWith(text, "0X")) { + uint64_t uint_val = 0; + if (absl::SimpleHexAtoi(text, &uint_val)) { + if (uint_val <= uint64_t{0x8000000000000000}) { + if (uint_val == uint64_t{0x8000000000000000}) { + int_val = std::numeric_limits::min(); + } else { + int_val = -static_cast(uint_val); + } + success = true; + } + } + } else { + if (absl::SimpleAtoi(text, &int_val)) { + int_val = -int_val; + success = true; + } else { + // Separately address -2^63, which is not representable as -(2^63) + std::string val = absl::StrCat("-", text); + success = absl::SimpleAtoi(val, &int_val); + } + } + if (success) { + return ast_factory_.NewIntConst(NextId(op.start), int_val); + } + ReportError(lit_tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); + } + if (peek_token_.type == TokenType::kFloat) { + Token lit_tok = NextToken(); + double double_val = 0.0; + if (absl::SimpleAtod(GetTokenText(lit_tok), &double_val)) { + return ast_factory_.NewDoubleConst(NextId(op.start), -double_val); + } + ReportError(lit_tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); + } + // Regular negate call + int64_t op_id = NextId(op); + ExprNode operand = ParseSelectorChain(); + return ast_factory_.NewCall(op_id, std::string(CelOperator::NEGATE), + std::vector{std::move(operand)}); + } + return ParsePrimary(); +} + +// Parses primary leaf expressions, including parenthesized expressions +// (`(expr)`), literals (`null`, `true`, numbers, strings, bytes), collection +// initializers +// (`[...]`, `{...}`), and identifiers/global function calls (`foo`, // `has(x.y)`). template ExprNode PrattParserWorker::ParsePrimary() { From ed733ed76c6e20a6e45b8dd82ae29f1328c8e2ab Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 24 Jul 2026 13:59:04 -0700 Subject: [PATCH 139/350] [Pratt Parser] Avoid deep-cloning cel::Expr PiperOrigin-RevId: 953542275 --- parser/internal/ast_factory.cc | 113 +++++++++++++++++--------- parser/internal/pratt_parser_worker.h | 48 +++++++---- 2 files changed, 104 insertions(+), 57 deletions(-) diff --git a/parser/internal/ast_factory.cc b/parser/internal/ast_factory.cc index 0c9a7d661..85e158a8e 100644 --- a/parser/internal/ast_factory.cc +++ b/parser/internal/ast_factory.cc @@ -139,113 +139,146 @@ absl::StatusOr AstFactoryInterface::CopyAndReplace( } std::optional replaced = replacer(expr); if (replaced.has_value()) { - return *replaced; + return std::move(*replaced); } - cel::Expr new_expr = expr; - switch (new_expr.kind_case()) { + cel::Expr new_expr; + new_expr.set_id(expr.id()); + + switch (expr.kind_case()) { case cel::ExprKindCase::kUnspecifiedExpr: + break; case cel::ExprKindCase::kConstant: + new_expr.set_const_expr(expr.const_expr()); + break; case cel::ExprKindCase::kIdentExpr: + new_expr.set_ident_expr(cel::IdentExpr(expr.ident_expr().name())); break; case cel::ExprKindCase::kSelectExpr: { cel::SelectExpr& select = new_expr.mutable_select_expr(); - if (select.has_operand()) { + select.set_field(expr.select_expr().field()); + select.set_test_only(expr.select_expr().test_only()); + if (expr.select_expr().has_operand()) { CEL_ASSIGN_OR_RETURN(cel::Expr operand, - CopyAndReplace(select.operand(), replacer, - max_recursion_depth - 1)); + CopyAndReplace(expr.select_expr().operand(), + replacer, max_recursion_depth - 1)); select.set_operand(std::move(operand)); } break; } case cel::ExprKindCase::kCallExpr: { cel::CallExpr& call = new_expr.mutable_call_expr(); - if (call.has_target()) { - CEL_ASSIGN_OR_RETURN( - cel::Expr target, - CopyAndReplace(call.target(), replacer, max_recursion_depth - 1)); + call.set_function(expr.call_expr().function()); + if (expr.call_expr().has_target()) { + CEL_ASSIGN_OR_RETURN(cel::Expr target, + CopyAndReplace(expr.call_expr().target(), replacer, + max_recursion_depth - 1)); call.set_target(std::move(target)); } - for (auto& arg : call.mutable_args()) { + call.mutable_args().reserve(expr.call_expr().args().size()); + for (const auto& arg : expr.call_expr().args()) { CEL_ASSIGN_OR_RETURN( cel::Expr new_arg, CopyAndReplace(arg, replacer, max_recursion_depth - 1)); - arg = std::move(new_arg); + call.mutable_args().push_back(std::move(new_arg)); } break; } case cel::ExprKindCase::kListExpr: { cel::ListExpr& list = new_expr.mutable_list_expr(); - for (auto& elem : list.mutable_elements()) { + list.mutable_elements().reserve(expr.list_expr().elements().size()); + for (const auto& elem : expr.list_expr().elements()) { + cel::ListExprElement new_elem; + new_elem.set_optional(elem.optional()); if (elem.has_expr()) { CEL_ASSIGN_OR_RETURN( - cel::Expr new_elem, + cel::Expr new_child, CopyAndReplace(elem.expr(), replacer, max_recursion_depth - 1)); - elem.set_expr(std::move(new_elem)); + new_elem.set_expr(std::move(new_child)); } + list.mutable_elements().push_back(std::move(new_elem)); } break; } case cel::ExprKindCase::kStructExpr: { cel::StructExpr& str = new_expr.mutable_struct_expr(); - for (auto& field : str.mutable_fields()) { + str.set_name(expr.struct_expr().name()); + str.mutable_fields().reserve(expr.struct_expr().fields().size()); + for (const auto& field : expr.struct_expr().fields()) { + cel::StructExprField new_field; + new_field.set_id(field.id()); + new_field.set_name(field.name()); + new_field.set_optional(field.optional()); if (field.has_value()) { CEL_ASSIGN_OR_RETURN( cel::Expr new_val, CopyAndReplace(field.value(), replacer, max_recursion_depth - 1)); - field.set_value(std::move(new_val)); + new_field.set_value(std::move(new_val)); } + str.mutable_fields().push_back(std::move(new_field)); } break; } case cel::ExprKindCase::kMapExpr: { cel::MapExpr& map = new_expr.mutable_map_expr(); - for (auto& entry : map.mutable_entries()) { + map.mutable_entries().reserve(expr.map_expr().entries().size()); + for (const auto& entry : expr.map_expr().entries()) { + cel::MapExprEntry new_entry; + new_entry.set_id(entry.id()); + new_entry.set_optional(entry.optional()); if (entry.has_key()) { CEL_ASSIGN_OR_RETURN( cel::Expr new_key, CopyAndReplace(entry.key(), replacer, max_recursion_depth - 1)); - entry.set_key(std::move(new_key)); + new_entry.set_key(std::move(new_key)); } if (entry.has_value()) { CEL_ASSIGN_OR_RETURN( cel::Expr new_val, CopyAndReplace(entry.value(), replacer, max_recursion_depth - 1)); - entry.set_value(std::move(new_val)); + new_entry.set_value(std::move(new_val)); } + map.mutable_entries().push_back(std::move(new_entry)); } break; } case cel::ExprKindCase::kComprehensionExpr: { cel::ComprehensionExpr& comp = new_expr.mutable_comprehension_expr(); - if (comp.has_accu_init()) { - CEL_ASSIGN_OR_RETURN(cel::Expr new_accu_init, - CopyAndReplace(comp.accu_init(), replacer, - max_recursion_depth - 1)); + comp.set_iter_var(expr.comprehension_expr().iter_var()); + comp.set_iter_var2(expr.comprehension_expr().iter_var2()); + comp.set_accu_var(expr.comprehension_expr().accu_var()); + if (expr.comprehension_expr().has_accu_init()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_accu_init, + CopyAndReplace(expr.comprehension_expr().accu_init(), replacer, + max_recursion_depth - 1)); comp.set_accu_init(std::move(new_accu_init)); } - if (comp.has_iter_range()) { - CEL_ASSIGN_OR_RETURN(cel::Expr new_iter_range, - CopyAndReplace(comp.iter_range(), replacer, - max_recursion_depth - 1)); + if (expr.comprehension_expr().has_iter_range()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_iter_range, + CopyAndReplace(expr.comprehension_expr().iter_range(), replacer, + max_recursion_depth - 1)); comp.set_iter_range(std::move(new_iter_range)); } - if (comp.has_loop_condition()) { - CEL_ASSIGN_OR_RETURN(cel::Expr new_loop_condition, - CopyAndReplace(comp.loop_condition(), replacer, - max_recursion_depth - 1)); + if (expr.comprehension_expr().has_loop_condition()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_loop_condition, + CopyAndReplace(expr.comprehension_expr().loop_condition(), replacer, + max_recursion_depth - 1)); comp.set_loop_condition(std::move(new_loop_condition)); } - if (comp.has_loop_step()) { - CEL_ASSIGN_OR_RETURN(cel::Expr new_loop_step, - CopyAndReplace(comp.loop_step(), replacer, - max_recursion_depth - 1)); + if (expr.comprehension_expr().has_loop_step()) { + CEL_ASSIGN_OR_RETURN( + cel::Expr new_loop_step, + CopyAndReplace(expr.comprehension_expr().loop_step(), replacer, + max_recursion_depth - 1)); comp.set_loop_step(std::move(new_loop_step)); } - if (comp.has_result()) { - CEL_ASSIGN_OR_RETURN( - cel::Expr new_result, - CopyAndReplace(comp.result(), replacer, max_recursion_depth - 1)); + if (expr.comprehension_expr().has_result()) { + CEL_ASSIGN_OR_RETURN(cel::Expr new_result, + CopyAndReplace(expr.comprehension_expr().result(), + replacer, max_recursion_depth - 1)); comp.set_result(std::move(new_result)); } break; diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index bca1de6b1..ad3bfda2d 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -322,10 +322,13 @@ ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { return lhs; } ExprNode false_expr = ParseBinaryAndTernary(0); - lhs = ast_factory_.NewCall( - op_id, CelOperator::CONDITIONAL, - std::vector{std::move(lhs), std::move(true_expr), - std::move(false_expr)}); + std::vector args; + args.reserve(3); + args.push_back(std::move(lhs)); + args.push_back(std::move(true_expr)); + args.push_back(std::move(false_expr)); + lhs = ast_factory_.NewCall(op_id, CelOperator::CONDITIONAL, + std::move(args)); continue; } @@ -340,9 +343,12 @@ ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { Token op_tok = NextToken(); int64_t op_id = NextId(op_tok); ExprNode rhs = ParseBinaryAndTernary(op_info.precedence + 1); - lhs = ast_factory_.NewCall( - op_id, std::string(op_info.name), - std::vector{std::move(lhs), std::move(rhs)}); + std::vector args; + args.reserve(2); + args.push_back(std::move(lhs)); + args.push_back(std::move(rhs)); + lhs = + ast_factory_.NewCall(op_id, std::string(op_info.name), std::move(args)); } return lhs; } @@ -396,11 +402,12 @@ ExprNode PrattParserWorker::ParseSelectorChain() { NormalizeIdent(id_tok, /*allow_quoted=*/!is_member_call); if (optional) { int64_t op_id = NextId(dot_tok); - lhs = ast_factory_.NewCall( - op_id, "_?._", - std::vector{ - std::move(lhs), - ast_factory_.NewStringConst(NextId(id_tok), id_text)}); + std::vector args; + args.reserve(2); + args.push_back(std::move(lhs)); + args.push_back( + ast_factory_.NewStringConst(NextId(id_tok), std::move(id_text))); + lhs = ast_factory_.NewCall(op_id, "_?._", std::move(args)); } else if (peek_token_.type == TokenType::kLeftParen) { Token lparen = NextToken(); int64_t call_id = NextId(lparen); @@ -429,9 +436,12 @@ ExprNode PrattParserWorker::ParseSelectorChain() { } ExprNode index = ParseExpr(); Expect(TokenType::kRightBracket, "expected ']'"); - lhs = ast_factory_.NewCall( - op_id, optional ? "_[?_]" : CelOperator::INDEX, - std::vector{std::move(lhs), std::move(index)}); + std::vector args; + args.reserve(2); + args.push_back(std::move(lhs)); + args.push_back(std::move(index)); + lhs = ast_factory_.NewCall(op_id, optional ? "_[?_]" : CelOperator::INDEX, + std::move(args)); } else if (tok == TokenType::kLeftBrace) { int32_t struct_pos = GetLeftmostPosition(lhs); if (auto struct_name = ExtractStructName(lhs); struct_name.has_value()) { @@ -455,8 +465,10 @@ ExprNode PrattParserWorker::ParseUnary() { Token op = NextToken(); int64_t op_id = NextId(op); ExprNode operand = ParseSelectorChain(); + std::vector args; + args.push_back(std::move(operand)); return ast_factory_.NewCall(op_id, std::string(CelOperator::LOGICAL_NOT), - std::vector{std::move(operand)}); + std::move(args)); } if (tok == TokenType::kMinus) { Token op = NextToken(); @@ -505,8 +517,10 @@ ExprNode PrattParserWorker::ParseUnary() { // Regular negate call int64_t op_id = NextId(op); ExprNode operand = ParseSelectorChain(); + std::vector args; + args.push_back(std::move(operand)); return ast_factory_.NewCall(op_id, std::string(CelOperator::NEGATE), - std::vector{std::move(operand)}); + std::move(args)); } return ParsePrimary(); } From 155ad1e48df48c7a7ba1d33c110fd7218fa9caca Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 24 Jul 2026 14:07:53 -0700 Subject: [PATCH 140/350] [Pratt Parser] Refactor worker methods and parse unary operator chains iteratively. - Extract helper methods (`ParseTernary`, `ParseSelectorChainTail`, `ParseUnaryOpsChain`, `ParseIdentOrCall`, `ParseNegativeIntLiteral`, `ParseNegativeDoubleLiteral`, `BuildBinaryCall`) to reduce the size of the call stack frames - Process prefix unary operator chains (`!`, `-`) iteratively in `ParseUnaryOpsChain` to reduce stack growth on deep chains. PiperOrigin-RevId: 953546973 --- parser/internal/BUILD | 1 + parser/internal/pratt_parser_test.cc | 48 ++++ parser/internal/pratt_parser_worker.h | 334 +++++++++++++++++--------- 3 files changed, 267 insertions(+), 116 deletions(-) diff --git a/parser/internal/BUILD b/parser/internal/BUILD index e3f2b8ed0..bf95d66b7 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -91,6 +91,7 @@ cc_library( "//internal:strings", "//parser:options", "//parser:parser_interface", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status:statusor", diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 26f849639..2956ec792 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -723,6 +723,18 @@ std::vector GetParserTestCases() { )^#4:Expr.Call# )", }, + TestCase{ + .source = "(((10 - 3) - 2))", + .expected_ast = R"( + _-_( + _-_( + 10^#1:int64#, + 3^#3:int64# + )^#2:Expr.Call#, + 2^#5:int64# + )^#4:Expr.Call# + )", + }, TestCase{ .source = "1 + 2 * 3 - 1 / 2 == 6 % 1", .expected_ast = R"( @@ -747,6 +759,30 @@ std::vector GetParserTestCases() { )^#10:Expr.Call# )", }, + TestCase{ + .source = "(1 + (2 * 3) - (1 / 2)) == (6 % 1)", + .expected_ast = R"( + _==_( + _-_( + _+_( + 1^#1:int64#, + _*_( + 2^#3:int64#, + 3^#5:int64# + )^#4:Expr.Call# + )^#2:Expr.Call#, + _/_( + 1^#7:int64#, + 2^#9:int64# + )^#8:Expr.Call# + )^#6:Expr.Call#, + _%_( + 6^#11:int64#, + 1^#13:int64# + )^#12:Expr.Call# + )^#10:Expr.Call# + )", + }, TestCase{ .source = "1 + 2 * 3 == 7 && true || false", .expected_ast = R"( @@ -1039,6 +1075,18 @@ std::vector GetParserTestCases() { )", .enable_optional_syntax = true, }, + TestCase{ + .source = "(((10 - 3) - 2))", + .expected_ast = R"( + _-_( + _-_( + 10^#1:int64#, + 3^#3:int64# + )^#2:Expr.Call#, + 2^#5:int64# + )^#4:Expr.Call# + )", + }, }; } diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index ad3bfda2d..8f44979c4 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -25,6 +25,7 @@ #include #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/container/flat_hash_map.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" @@ -179,7 +180,6 @@ class PrattParserWorker : public ParserWorker { using CelOperator = ::google::api::expr::common::CelOperator; ExprNode ParseExpr(); - // Parses binary operator expressions and ternary conditional expressions // (`? :`) using operator-precedence (Pratt) parsing. Consumes operators from // the token stream whose binding precedence is greater than or equal to @@ -194,6 +194,10 @@ class PrattParserWorker : public ParserWorker { // and `ParseBinary(0)` for false branch `c`. ExprNode ParseBinaryAndTernary(int min_prec); + // Parses ternary conditional expressions (`condition ? true_expr : + // false_expr`). + ExprNode ParseTernary(ExprNode lhs); + // Helper method for parsing a contiguous chain of same-precedence logical // operators (`&&` or `||`) iteratively into a list of terms and operator IDs. // Constructs either a balanced binary AST (`(a && b) && c`) or a single @@ -216,6 +220,13 @@ class PrattParserWorker : public ParserWorker { // then loops iteratively through `.b`, `[0]`, and `.c(x)`. ExprNode ParseSelectorChain(); + // Handler for postfix member, index, receiver method, and + // struct initializer operations (`.field`, `[index]`, `.method(args)`, + // `Type{field: val}`). + // + // Processes continuous postfix operation chains iteratively. + ExprNode ParseSelectorChainTail(ExprNode lhs); + // Parses prefix unary operators (logical NOT `!` and negation `-`). If a // numeric literal immediately follows `-`, folds it directly into a negative // constant node (`-42`, `-3.14`). Otherwise, wraps the operand in a `_!_` or @@ -228,6 +239,9 @@ class PrattParserWorker : public ParserWorker { // wrapping `has(x.y)`. ExprNode ParseUnary(); + // Parses unary operator chains (`!`, `-`). + ExprNode ParseUnaryOpsChain(Token first_op); + // Parses primary leaf expressions (`nud` atomic atoms), including // parenthesized expressions (`(expr)`), literal constants (`null`, `true`, // `false`, numbers, strings, bytes), collection initializers (`[list]`, @@ -250,10 +264,15 @@ class PrattParserWorker : public ParserWorker { ExprNode ParseStruct(int64_t obj_id, absl::string_view struct_name); std::vector ParseArguments(TokenType close_token); ExprNode ParseIntLiteral(); + ExprNode ParseNegativeIntLiteral(int64_t node_id); ExprNode ParseUintLiteral(); ExprNode ParseDoubleLiteral(); + ExprNode ParseNegativeDoubleLiteral(int64_t node_id); ExprNode ParseStringLiteral(); ExprNode ParseBytesLiteral(); + ExprNode BuildBinaryCall(int64_t op_id, absl::string_view op_name, + ExprNode lhs, ExprNode rhs); + ExprNode ParseIdentOrCall(); std::string NormalizeIdent(const Token& tok, bool allow_quoted); std::optional ExtractStructName(const ExprNode& expr); int32_t GetLeftmostPosition(const ExprNode& expr); @@ -304,8 +323,37 @@ ExprNode PrattParserWorker::ParseExpr() { return expr; } +template +ExprNode PrattParserWorker::ParseTernary(ExprNode lhs) { + NextToken(); + int64_t op_id = NextId(); + ExprNode true_expr = ParseBinaryAndTernary(1); + if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { + return lhs; + } + ExprNode false_expr = ParseBinaryAndTernary(0); + std::vector args; + args.reserve(3); + args.push_back(std::move(lhs)); + args.push_back(std::move(true_expr)); + args.push_back(std::move(false_expr)); + return ast_factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(args)); +} + const BinaryOpInfo& GetBinaryOpInfo(TokenType type); +template +ExprNode PrattParserWorker::BuildBinaryCall(int64_t op_id, + absl::string_view op_name, + ExprNode lhs, + ExprNode rhs) { + std::vector args; + args.reserve(2); + args.push_back(std::move(lhs)); + args.push_back(std::move(rhs)); + return ast_factory_.NewCall(op_id, std::string(op_name), std::move(args)); +} + // Parses binary operator expressions and ternary conditional expressions // (`? :`) using Pratt operator-precedence parsing (e.g., `a + b * c`). template @@ -314,21 +362,7 @@ ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { while (true) { TokenType tok = peek_token_.type; if (tok == TokenType::kQuestion && min_prec <= 0) { - NextToken(); - int64_t op_id = NextId(); - ExprNode true_expr = ParseBinaryAndTernary(1); - if (!Expect(TokenType::kColon, - "expected ':' in conditional expression")) { - return lhs; - } - ExprNode false_expr = ParseBinaryAndTernary(0); - std::vector args; - args.reserve(3); - args.push_back(std::move(lhs)); - args.push_back(std::move(true_expr)); - args.push_back(std::move(false_expr)); - lhs = ast_factory_.NewCall(op_id, CelOperator::CONDITIONAL, - std::move(args)); + lhs = ParseTernary(std::move(lhs)); continue; } @@ -343,12 +377,7 @@ ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { Token op_tok = NextToken(); int64_t op_id = NextId(op_tok); ExprNode rhs = ParseBinaryAndTernary(op_info.precedence + 1); - std::vector args; - args.reserve(2); - args.push_back(std::move(lhs)); - args.push_back(std::move(rhs)); - lhs = - ast_factory_.NewCall(op_id, std::string(op_info.name), std::move(args)); + lhs = BuildBinaryCall(op_id, op_info.name, std::move(lhs), std::move(rhs)); } return lhs; } @@ -371,11 +400,21 @@ ExprNode PrattParserWorker::ParseBalancedLogicalChain( options_.enable_variadic_logical_operators); } -// Parses prefix and postfix member/indexing operations iteratively -// (e.g., `!a.b[0].c(x)`). template ExprNode PrattParserWorker::ParseSelectorChain() { ExprNode lhs = ParseUnary(); + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || + tok == TokenType::kLeftBrace) { + return ParseSelectorChainTail(std::move(lhs)); + } + return lhs; +} + +// Parses prefix and postfix member/indexing operations iteratively +// (e.g., `!a.b[0].c(x)`). +template +ExprNode PrattParserWorker::ParseSelectorChainTail(ExprNode lhs) { while (true) { TokenType tok = peek_token_.type; if (tok == TokenType::kDot) { @@ -456,73 +495,124 @@ ExprNode PrattParserWorker::ParseSelectorChain() { return lhs; } -// Parses prefix unary operators (`!`, `-`) and folds negative numeric literals -// (e.g., `-42`, `!has(x.y)`). template -ExprNode PrattParserWorker::ParseUnary() { - TokenType tok = peek_token_.type; - if (tok == TokenType::kExclamation) { +ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { + struct UnaryOpInfo { + TokenType type; + int64_t id; + }; + std::vector ops; + ops.push_back({first_op.type, NextId(first_op)}); + + while (peek_token_.type == TokenType::kExclamation || + peek_token_.type == TokenType::kMinus) { Token op = NextToken(); - int64_t op_id = NextId(op); - ExprNode operand = ParseSelectorChain(); + ops.push_back({op.type, NextId(op)}); + } + + ExprNode operand; + if (!ops.empty() && ops.back().type == TokenType::kMinus) { + if (peek_token_.type == TokenType::kInt) { + int64_t op_id = ops.back().id; + ops.pop_back(); + operand = ParseNegativeIntLiteral(op_id); + } else if (peek_token_.type == TokenType::kFloat) { + int64_t op_id = ops.back().id; + ops.pop_back(); + operand = ParseNegativeDoubleLiteral(op_id); + } else { + operand = ParseSelectorChain(); + } + } else { + operand = ParseSelectorChain(); + } + + for (int i = static_cast(ops.size()) - 1; i >= 0; --i) { std::vector args; args.push_back(std::move(operand)); - return ast_factory_.NewCall(op_id, std::string(CelOperator::LOGICAL_NOT), - std::move(args)); + absl::string_view op_name = (ops[i].type == TokenType::kExclamation) + ? CelOperator::LOGICAL_NOT + : CelOperator::NEGATE; + operand = + ast_factory_.NewCall(ops[i].id, std::string(op_name), std::move(args)); } - if (tok == TokenType::kMinus) { - Token op = NextToken(); + + return operand; +} + +// Parses prefix unary operators (`!`, `-`) iteratively and folds negative +// numeric literals (e.g., `-42`, `!has(x.y)`). +template +ExprNode PrattParserWorker::ParseUnary() { + TokenType tok = peek_token_.type; + if (tok != TokenType::kExclamation && tok != TokenType::kMinus) { + return ParsePrimary(); + } + + Token op = NextToken(); + TokenType op_type = op.type; + if (peek_token_.type == TokenType::kExclamation || + peek_token_.type == TokenType::kMinus) { + return ParseUnaryOpsChain(op); + } + + if (op_type == TokenType::kMinus) { if (peek_token_.type == TokenType::kInt) { - Token lit_tok = NextToken(); - std::string text = GetTokenText(lit_tok); - int64_t int_val = 0; - bool success = false; - if (absl::StartsWith(text, "0x") || absl::StartsWith(text, "0X")) { - uint64_t uint_val = 0; - if (absl::SimpleHexAtoi(text, &uint_val)) { - if (uint_val <= uint64_t{0x8000000000000000}) { - if (uint_val == uint64_t{0x8000000000000000}) { - int_val = std::numeric_limits::min(); - } else { - int_val = -static_cast(uint_val); - } - success = true; - } - } - } else { - if (absl::SimpleAtoi(text, &int_val)) { - int_val = -int_val; - success = true; - } else { - // Separately address -2^63, which is not representable as -(2^63) - std::string val = absl::StrCat("-", text); - success = absl::SimpleAtoi(val, &int_val); - } - } - if (success) { - return ast_factory_.NewIntConst(NextId(op.start), int_val); - } - ReportError(lit_tok, "invalid int literal"); - return ast_factory_.NewUnspecified(NextId(lit_tok)); + return ParseNegativeIntLiteral(NextId(op)); } if (peek_token_.type == TokenType::kFloat) { - Token lit_tok = NextToken(); - double double_val = 0.0; - if (absl::SimpleAtod(GetTokenText(lit_tok), &double_val)) { - return ast_factory_.NewDoubleConst(NextId(op.start), -double_val); - } - ReportError(lit_tok, "invalid double literal"); - return ast_factory_.NewUnspecified(NextId(lit_tok)); + return ParseNegativeDoubleLiteral(NextId(op)); } - // Regular negate call - int64_t op_id = NextId(op); - ExprNode operand = ParseSelectorChain(); - std::vector args; - args.push_back(std::move(operand)); - return ast_factory_.NewCall(op_id, std::string(CelOperator::NEGATE), - std::move(args)); } - return ParsePrimary(); + + int64_t op_id = NextId(op); + ExprNode operand = ParseSelectorChain(); + std::vector args; + args.push_back(std::move(operand)); + absl::string_view op_name = (op_type == TokenType::kExclamation) + ? CelOperator::LOGICAL_NOT + : CelOperator::NEGATE; + return ast_factory_.NewCall(op_id, std::string(op_name), std::move(args)); +} + +// Parses identifiers (e.g., `foo`, `.foo`) and global function or macro calls +// (e.g., `foo(args)`, `has(x.y)`). +template +ExprNode PrattParserWorker::ParseIdentOrCall() { + TokenType tok_type = peek_token_.type; + bool leading_dot = false; + Token first_tok = peek_token_; + if (tok_type == TokenType::kDot) { + NextToken(); + leading_dot = true; + } + Token id_tok = NextToken(); + if (id_tok.type != TokenType::kIdent && + id_tok.type != TokenType::kReservedWord) { + if (id_tok.type != TokenType::kError) { + ReportError(id_tok, "expected identifier"); + } + return ast_factory_.NewUnspecified(NextId(id_tok)); + } + std::string id_text = NormalizeIdent(id_tok, /*allow_quoted=*/false); + if (ABSL_PREDICT_FALSE(id_tok.type == TokenType::kReservedWord)) { + if (cel::internal::LexisIsReserved(id_text)) { + ReportError(id_tok, absl::StrFormat("reserved identifier: %s", id_text)); + } + } + std::string name = + leading_dot ? absl::StrCat(".", id_text) : std::string(id_text); + int64_t id = NextId(leading_dot ? first_tok : id_tok); + if (peek_token_.type == TokenType::kLeftParen) { + NextToken(); + std::vector args = ParseArguments(TokenType::kRightParen); + if (auto expanded = TryExpandMacro(id, name, nullptr, args); + expanded.has_value()) { + return std::move(*expanded); + } + return ast_factory_.NewCall(id, name, std::move(args)); + } + return ast_factory_.NewIdent(id, std::move(name)); } // Parses primary leaf expressions, including parenthesized expressions @@ -560,41 +650,7 @@ ExprNode PrattParserWorker::ParsePrimary() { expr = ParseMap(); } else if (tok_type == TokenType::kDot || tok_type == TokenType::kIdent || tok_type == TokenType::kReservedWord) { - bool leading_dot = false; - Token first_tok = peek_token_; - if (tok_type == TokenType::kDot) { - NextToken(); - leading_dot = true; - } - Token id_tok = NextToken(); - if (id_tok.type != TokenType::kIdent && - id_tok.type != TokenType::kReservedWord) { - if (id_tok.type != TokenType::kError) { - ReportError(id_tok, "expected identifier"); - } - expr = ast_factory_.NewUnspecified(NextId(id_tok)); - } else { - std::string id_text = NormalizeIdent(id_tok, /*allow_quoted=*/false); - if (cel::internal::LexisIsReserved(id_text)) { - ReportError(id_tok, - absl::StrFormat("reserved identifier: %s", id_text)); - } - std::string name = - leading_dot ? absl::StrCat(".", id_text) : std::string(id_text); - int64_t id = NextId(leading_dot ? first_tok : id_tok); - if (peek_token_.type == TokenType::kLeftParen) { - NextToken(); - std::vector args = ParseArguments(TokenType::kRightParen); - if (auto expanded = TryExpandMacro(id, name, nullptr, args); - expanded.has_value()) { - expr = std::move(*expanded); - } else { - expr = ast_factory_.NewCall(id, name, std::move(args)); - } - } else { - expr = ast_factory_.NewIdent(id, std::move(name)); - } - } + expr = ParseIdentOrCall(); } else { Token bad_tok = NextToken(); if (bad_tok.type != TokenType::kError) { @@ -757,6 +813,40 @@ ExprNode PrattParserWorker::ParseIntLiteral() { return ast_factory_.NewUnspecified(NextId(tok)); } +template +ExprNode PrattParserWorker::ParseNegativeIntLiteral(int64_t node_id) { + Token lit_tok = NextToken(); + std::string text = GetTokenText(lit_tok); + int64_t int_val = 0; + bool success = false; + if (absl::StartsWith(text, "0x") || absl::StartsWith(text, "0X")) { + uint64_t uint_val = 0; + if (absl::SimpleHexAtoi(text, &uint_val)) { + if (uint_val <= uint64_t{0x8000000000000000}) { + if (uint_val == uint64_t{0x8000000000000000}) { + int_val = std::numeric_limits::min(); + } else { + int_val = -static_cast(uint_val); + } + success = true; + } + } + } else if (absl::SimpleAtoi(text, &int_val)) { + int_val = -int_val; + success = true; + } else { + // Separately handle -2^63, which is not representable as -(2^63) + std::string val = absl::StrCat("-", text); + success = absl::SimpleAtoi(val, &int_val); + } + + if (success) { + return ast_factory_.NewIntConst(node_id, int_val); + } + ReportError(lit_tok, "invalid int literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); +} + // Parses unsigned ints (e.g., `42u`, `0x1Au`). template ExprNode PrattParserWorker::ParseUintLiteral() { @@ -790,6 +880,18 @@ ExprNode PrattParserWorker::ParseDoubleLiteral() { return ast_factory_.NewUnspecified(NextId(tok)); } +template +ExprNode PrattParserWorker::ParseNegativeDoubleLiteral( + int64_t node_id) { + Token lit_tok = NextToken(); + double double_val = 0.0; + if (absl::SimpleAtod(GetTokenText(lit_tok), &double_val)) { + return ast_factory_.NewDoubleConst(node_id, -double_val); + } + ReportError(lit_tok, "invalid double literal"); + return ast_factory_.NewUnspecified(NextId(lit_tok)); +} + // Parses string literals (e.g., `"hello"`, `'world'`, `"""multi"""`). template ExprNode PrattParserWorker::ParseStringLiteral() { From 30c794bc07fbf2fa9b1f482ead8213cf1c43ce90 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 24 Jul 2026 14:16:36 -0700 Subject: [PATCH 141/350] [Pratt Parser] Inline the ParsePrimary function PiperOrigin-RevId: 953551347 --- parser/internal/pratt_parser_worker.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 8f44979c4..b022ae9ee 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -24,6 +24,7 @@ #include #include +#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/container/flat_hash_map.h" @@ -257,7 +258,7 @@ class PrattParserWorker : public ParserWorker { // Example (`(a + b)`): Consumes `(`, recurses to `ParseExpr()`, and expects // `)`. Example (`has(x.y)`): Consumes `has`, parses arguments `(x.y)`, and // expands the `has` macro. - ExprNode ParsePrimary(); + ABSL_ATTRIBUTE_ALWAYS_INLINE inline ExprNode ParsePrimary(); ExprNode ParseList(); ExprNode ParseMap(); @@ -621,7 +622,8 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { // (`[...]`, `{...}`), and identifiers/global function calls (`foo`, // `has(x.y)`). template -ExprNode PrattParserWorker::ParsePrimary() { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline ExprNode +PrattParserWorker::ParsePrimary() { ExprNode expr; TokenType tok_type = peek_token_.type; if (tok_type == TokenType::kLeftParen) { From 5cf72d916e6fe23a75b791f8762ba99ebc698bd2 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 24 Jul 2026 14:48:37 -0700 Subject: [PATCH 142/350] [Pratt Parser] Add an option to enable Pratt Parser PiperOrigin-RevId: 953565535 --- conformance/BUILD | 4 +- conformance/run.bzl | 51 +-- conformance/run.cc | 3 + conformance/service.cc | 39 ++- conformance/service.h | 1 + parser/BUILD | 4 +- parser/internal/BUILD | 1 + parser/internal/ast_factory_test.cc | 4 +- parser/internal/pratt_parser.cc | 27 +- parser/internal/pratt_parser.h | 5 + parser/internal/pratt_parser_test.cc | 4 +- parser/options.h | 11 + parser/parser.cc | 24 +- parser/parser_test.cc | 494 ++++++++++++++++++++------- 14 files changed, 502 insertions(+), 170 deletions(-) diff --git a/conformance/BUILD b/conformance/BUILD index d72237841..eb129470f 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -248,7 +248,7 @@ _TESTS_TO_SKIP_LEGACY_DASHBOARD = [ ] # Generates a bunch of `cc_test` whose names follow the pattern -# `conformance_(...)_{arena|refcount}_{optimized|unoptimized}_{recursive|iterative}`. +# `conformance_(...)_{pratt|antlr}_{optimized|unoptimized}_{recursive|iterative}`. gen_conformance_tests( name = "conformance_parse_only", data = _ALL_TESTS, @@ -317,7 +317,7 @@ gen_conformance_tests( ) # Generates a bunch of `cc_test` whose names follow the pattern -# `conformance_dashboard_..._{arena|refcount}_{optimized|unoptimized}_{recursive|iterative}`. +# `conformance_dashboard_..._{pratt|antlr}_{optimized|unoptimized}_{recursive|iterative}`. gen_conformance_tests( name = "conformance_dashboard_parse_only", dashboard = True, diff --git a/conformance/run.bzl b/conformance/run.bzl index 8faeb6c16..76b33efb3 100644 --- a/conformance/run.bzl +++ b/conformance/run.bzl @@ -47,16 +47,17 @@ def _expand_tests_to_skip(tests_to_skip): result.append(test_to_skip[0:slash] + part) return result -def _conformance_test_name(name, optimize, recursive): +def _conformance_test_name(name, pratt, optimize, recursive): return "_".join( [ name, + "pratt" if pratt else "antlr", "optimized" if optimize else "unoptimized", "recursive" if recursive else "iterative", ], ) -def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators): +def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators, pratt): args = [] if modern: args.append("--modern") @@ -74,12 +75,16 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, args.append("--dashboard") if enable_variadic_logical_operators: args.append("--enable_variadic_logical_operators") + if pratt: + args.append("--enable_pratt_parser") + else: + args.append("--noenable_pratt_parser") return args -def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard, enable_variadic_logical_operators): +def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard, enable_variadic_logical_operators, pratt): cc_test( - name = _conformance_test_name(name, optimize, recursive), - args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators) + ["$(rlocationpath {})".format(test) for test in data], + name = _conformance_test_name(name, pratt, optimize, recursive), + args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators, pratt) + ["$(rlocationpath {})".format(test) for test in data], env = select( { "@platforms//os:windows": {"CEL_SKIP_TESTS": ",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS)}, @@ -108,23 +113,25 @@ def gen_conformance_tests(name, data, modern = False, checked = False, select_op """ skip_check = not checked tests = [] - for optimize in (True, False): - for recursive in (True, False): - test_name = _conformance_test_name(name, optimize, recursive) - tests.append(test_name) - _conformance_test( - name, - data, - modern = modern, - optimize = optimize, - recursive = recursive, - select_opt = select_opt, - skip_check = skip_check, - skip_tests = _expand_tests_to_skip(skip_tests), - tags = tags, - dashboard = dashboard, - enable_variadic_logical_operators = enable_variadic_logical_operators, - ) + for pratt in (True, False): + for optimize in (True, False): + for recursive in (True, False): + test_name = _conformance_test_name(name, pratt, optimize, recursive) + tests.append(test_name) + _conformance_test( + name, + data, + modern = modern, + optimize = optimize, + recursive = recursive, + select_opt = select_opt, + skip_check = skip_check, + skip_tests = _expand_tests_to_skip(skip_tests), + tags = tags, + dashboard = dashboard, + enable_variadic_logical_operators = enable_variadic_logical_operators, + pratt = pratt, + ) native.test_suite( name = name, tests = tests, diff --git a/conformance/run.cc b/conformance/run.cc index 1be16ba60..6b7bae071 100644 --- a/conformance/run.cc +++ b/conformance/run.cc @@ -69,6 +69,8 @@ ABSL_FLAG(bool, select_optimization, false, "Enable select optimization."); ABSL_FLAG(bool, enable_variadic_logical_operators, false, "Enable parsing logical AND & OR operators as a single flat variadic " "call."); +ABSL_FLAG(bool, enable_pratt_parser, true, + "Enable manual (Pratt) parser instead of ANTLR parser."); namespace { @@ -266,6 +268,7 @@ NewConformanceServiceFromFlags() { .select_optimization = absl::GetFlag(FLAGS_select_optimization), .enable_variadic_logical_operators = absl::GetFlag(FLAGS_enable_variadic_logical_operators), + .enable_pratt_parser = absl::GetFlag(FLAGS_enable_pratt_parser), }); ABSL_CHECK_OK(status_or_service); return std::shared_ptr( diff --git a/conformance/service.cc b/conformance/service.cc index 4d3815d37..8a456a9f4 100644 --- a/conformance/service.cc +++ b/conformance/service.cc @@ -130,7 +130,8 @@ cel::expr::Expr ExtractExpr( absl::Status LegacyParse(const conformance::v1alpha1::ParseRequest& request, conformance::v1alpha1::ParseResponse& response, bool enable_optional_syntax, - bool enable_variadic_logical_operators) { + bool enable_variadic_logical_operators, + bool enable_pratt_parser) { if (request.cel_source().empty()) { return absl::InvalidArgumentError("no source code"); } @@ -138,6 +139,7 @@ absl::Status LegacyParse(const conformance::v1alpha1::ParseRequest& request, options.enable_optional_syntax = enable_optional_syntax; options.enable_quoted_identifiers = true; options.enable_variadic_logical_operators = enable_variadic_logical_operators; + options.enable_pratt_parser = enable_pratt_parser; cel::MacroRegistry macros; CEL_RETURN_IF_ERROR(cel::RegisterStandardMacros(macros, options)); CEL_RETURN_IF_ERROR( @@ -240,7 +242,7 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { public: static absl::StatusOr> Create( bool optimize, bool recursive, bool select_optimization, - bool enable_variadic_logical_operators) { + bool enable_variadic_logical_operators, bool enable_pratt_parser) { static auto* constant_arena = new Arena(); google::protobuf::LinkMessageReflection< @@ -319,14 +321,15 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { builder->GetRegistry(), options)); return absl::WrapUnique(new LegacyConformanceServiceImpl( - std::move(builder), enable_variadic_logical_operators)); + std::move(builder), enable_variadic_logical_operators, + enable_pratt_parser)); } void Parse(const conformance::v1alpha1::ParseRequest& request, conformance::v1alpha1::ParseResponse& response) override { auto status = LegacyParse(request, response, /*enable_optional_syntax=*/false, - enable_variadic_logical_operators_); + enable_variadic_logical_operators_, enable_pratt_parser_); if (!status.ok()) { auto* issue = response.add_issues(); issue->set_code(ToGrpcCode(status.code())); @@ -425,19 +428,22 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface { private: LegacyConformanceServiceImpl(std::unique_ptr builder, - bool enable_variadic_logical_operators) + bool enable_variadic_logical_operators, + bool enable_pratt_parser) : builder_(std::move(builder)), - enable_variadic_logical_operators_(enable_variadic_logical_operators) {} + enable_variadic_logical_operators_(enable_variadic_logical_operators), + enable_pratt_parser_(enable_pratt_parser) {} std::unique_ptr builder_; bool enable_variadic_logical_operators_; + bool enable_pratt_parser_; }; class ModernConformanceServiceImpl : public ConformanceServiceInterface { public: static absl::StatusOr> Create( bool optimize, bool recursive, bool select_optimization, - bool enable_variadic_logical_operators) { + bool enable_variadic_logical_operators, bool enable_pratt_parser) { google::protobuf::LinkMessageReflection< cel::expr::conformance::proto3::TestAllTypes>(); google::protobuf::LinkMessageReflection< @@ -479,9 +485,9 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { options.max_recursion_depth = 48; } - return absl::WrapUnique( - new ModernConformanceServiceImpl(options, optimize, select_optimization, - enable_variadic_logical_operators)); + return absl::WrapUnique(new ModernConformanceServiceImpl( + options, optimize, select_optimization, + enable_variadic_logical_operators, enable_pratt_parser)); } absl::StatusOr> Setup( @@ -538,7 +544,7 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { conformance::v1alpha1::ParseResponse& response) override { auto status = LegacyParse(request, response, /*enable_optional_syntax=*/true, - enable_variadic_logical_operators_); + enable_variadic_logical_operators_, enable_pratt_parser_); if (!status.ok()) { auto* issue = response.add_issues(); issue->set_code(ToGrpcCode(status.code())); @@ -630,11 +636,13 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { ModernConformanceServiceImpl(const RuntimeOptions& options, bool enable_optimizations, bool enable_select_optimization, - bool enable_variadic_logical_operators) + bool enable_variadic_logical_operators, + bool enable_pratt_parser) : options_(options), enable_optimizations_(enable_optimizations), enable_select_optimization_(enable_select_optimization), - enable_variadic_logical_operators_(enable_variadic_logical_operators) {} + enable_variadic_logical_operators_(enable_variadic_logical_operators), + enable_pratt_parser_(enable_pratt_parser) {} static absl::StatusOr> Plan( const cel::Runtime& runtime, @@ -666,6 +674,7 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { bool enable_optimizations_; bool enable_select_optimization_; bool enable_variadic_logical_operators_; + bool enable_pratt_parser_; }; } // namespace @@ -679,11 +688,11 @@ NewConformanceService(const ConformanceServiceOptions& options) { if (options.modern) { return google::api::expr::runtime::ModernConformanceServiceImpl::Create( options.optimize, options.recursive, options.select_optimization, - options.enable_variadic_logical_operators); + options.enable_variadic_logical_operators, options.enable_pratt_parser); } else { return google::api::expr::runtime::LegacyConformanceServiceImpl::Create( options.optimize, options.recursive, options.select_optimization, - options.enable_variadic_logical_operators); + options.enable_variadic_logical_operators, options.enable_pratt_parser); } } diff --git a/conformance/service.h b/conformance/service.h index 8eb97296e..4cf2aec73 100644 --- a/conformance/service.h +++ b/conformance/service.h @@ -47,6 +47,7 @@ struct ConformanceServiceOptions { bool recursive; bool select_optimization; bool enable_variadic_logical_operators = false; + bool enable_pratt_parser = true; }; absl::StatusOr> diff --git a/parser/BUILD b/parser/BUILD index 6650d9fe9..04a2c0516 100644 --- a/parser/BUILD +++ b/parser/BUILD @@ -42,6 +42,7 @@ cc_library( ":source_factory", "//common:ast", "//common:constant", + "//common:expr", "//common:expr_factory", "//common:operators", "//common:source", @@ -50,8 +51,8 @@ cc_library( "//internal:lexis", "//internal:status_macros", "//internal:strings", - "//internal:utf8", "//parser/internal:cel_cc_parser", + "//parser/internal:pratt_parser", "@antlr4-cpp-runtime", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/cleanup", @@ -60,6 +61,7 @@ cc_library( "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:check", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/parser/internal/BUILD b/parser/internal/BUILD index bf95d66b7..8638fcc92 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -117,6 +117,7 @@ cc_library( "//parser:macro_registry", "//parser:options", "//parser:parser_interface", + "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", diff --git a/parser/internal/ast_factory_test.cc b/parser/internal/ast_factory_test.cc index 3e80a4a17..585c1a269 100644 --- a/parser/internal/ast_factory_test.cc +++ b/parser/internal/ast_factory_test.cc @@ -405,8 +405,8 @@ class TestMacroExprExpanderSupport public: int64_t NextId() override { return 42; } int64_t CopyId(int64_t id) override { return id; } - cel::Expr ReportError(std::string_view) override { return cel::Expr(); } - cel::Expr ReportErrorAt(const cel::Expr&, std::string_view) override { + cel::Expr ReportError(absl::string_view) override { return cel::Expr(); } + cel::Expr ReportErrorAt(const cel::Expr&, absl::string_view) override { return cel::Expr(); } }; diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc index e83c21fce..0ccc28c15 100644 --- a/parser/internal/pratt_parser.cc +++ b/parser/internal/pratt_parser.cc @@ -14,6 +14,7 @@ #include "parser/internal/pratt_parser.h" +#include #include #include #include @@ -21,6 +22,7 @@ #include #include +#include "absl/algorithm/container.h" #include "absl/base/nullability.h" #include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_map.h" @@ -158,21 +160,34 @@ template class PrattParserWorker; absl::StatusOr> PrattParserImpl::ParseImpl( const cel::Source& source, std::vector* absl_nullable parse_issues) const { - if (source.content().size() > options_.expression_size_codepoint_limit) { + return PrattParseImpl(source, macro_registry_, options_, parse_issues); +} + +absl::StatusOr> PrattParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, std::vector* parse_issues) { + if (source.content().size() > options.expression_size_codepoint_limit) { return absl::InvalidArgumentError(absl::StrFormat( - "expression size exceeds codepoint limit: %zu > %d", - source.content().size(), options_.expression_size_codepoint_limit)); + "expression size exceeds codepoint limit. input size: %zu, limit: %d", + source.content().size(), options.expression_size_codepoint_limit)); } std::vector issues; - AstFactory factory(¯o_registry_); - PrattParserWorker worker(source, options_, &issues, factory); + AstFactory factory(®istry); + PrattParserWorker worker(source, options, &issues, factory); Expr expr = worker.Parse(); if (worker.is_recursion_limit_exceeded()) { return absl::CancelledError( absl::StrFormat("Expression recursion limit exceeded. limit: %d", - options_.max_recursion_depth)); + options.max_recursion_depth)); } if (worker.has_errors()) { + absl::c_stable_sort( + issues, [](const cel::ParseIssue& lhs, const cel::ParseIssue& rhs) { + if (lhs.location().line != rhs.location().line) { + return lhs.location().line < rhs.location().line; + } + return lhs.location().column < rhs.location().column; + }); std::string err_msg = FormatIssues(source, issues); if (parse_issues != nullptr) { parse_issues->swap(issues); diff --git a/parser/internal/pratt_parser.h b/parser/internal/pratt_parser.h index 21e9878f3..469f48aec 100644 --- a/parser/internal/pratt_parser.h +++ b/parser/internal/pratt_parser.h @@ -67,6 +67,11 @@ class PrattParserImpl final : public cel::Parser { absl::flat_hash_set library_ids_; }; +absl::StatusOr> PrattParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, + std::vector* parse_issues = nullptr); + class PrattParserBuilderImpl final : public cel::ParserBuilder { public: explicit PrattParserBuilderImpl(const cel::ParserOptions& options) diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 2956ec792..7b2570514 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -1810,10 +1810,10 @@ TEST(PrattParserErrorRecoveryTest, ErrorRecoveryLimitOne) { EXPECT_THAT(result, StatusIs(absl::StatusCode::kInvalidArgument)); ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("......")); EXPECT_EQ(FormatIssues(*source, issues), + "ERROR: :-1:0: Error recovery limit (1) exceeded\n" "ERROR: :1:2: expected identifier\n" " | ......\n" - " | .^\n" - "ERROR: :-1:0: Error recovery limit (1) exceeded"); + " | .^"); } } // namespace diff --git a/parser/options.h b/parser/options.h index 22154e1f9..eca41810a 100644 --- a/parser/options.h +++ b/parser/options.h @@ -66,6 +66,17 @@ struct ParserOptions final { // Enables parsing logical AND & OR operators as a single flat variadic call // instead of a balanced/nested binary AST structure. bool enable_variadic_logical_operators = false; + + // Enable the manual (Pratt) parser instead of the ANTLR parser. + // + // CAUTION: Pratt Parser is work in progress. Don't use unless you are aware + // of the risks. There is currently no support whatsoever available for + // clients of this option. + // + // This option is temporary and should not be used by new code outside of the + // early testing of the Pratt parser. + // TODO(b/527638023): Remove this option once the ANTLR parser is removed. + bool enable_pratt_parser = false; }; } // namespace cel diff --git a/parser/parser.cc b/parser/parser.cc index 24b4ca079..6a66ffc9a 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -25,8 +25,9 @@ #include #include #include +#include #include -#include +#include #include #include @@ -39,6 +40,7 @@ #include "absl/container/flat_hash_set.h" #include "absl/functional/overload.h" #include "absl/log/absl_check.h" +#include "absl/log/check.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -57,13 +59,14 @@ #include "common/ast/expr_proto.h" #include "common/ast/source_info_proto.h" #include "common/constant.h" +#include "common/expr.h" #include "common/expr_factory.h" #include "common/operators.h" #include "common/source.h" #include "internal/lexis.h" #include "internal/status_macros.h" #include "internal/strings.h" -#include "internal/utf8.h" +#include "parser/internal/pratt_parser.h" #pragma push_macro("IN") #undef IN #include "parser/internal/CelBaseVisitor.h" @@ -1661,6 +1664,7 @@ absl::StatusOr ParseImpl( const cel::Source& source, const cel::MacroRegistry& registry, const ParserOptions& options, std::vector* parse_issues = nullptr) { + ABSL_DCHECK(!options.enable_pratt_parser); try { CodePointStream input(source.content(), source.description()); if (input.size() > options.expression_size_codepoint_limit) { @@ -1844,6 +1848,10 @@ class ParserBuilderImpl : public cel::ParserBuilder { library_ids.insert("optional"); } CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); + if (options_.enable_pratt_parser) { + return std::make_unique( + options_, std::move(macro_registry), std::move(library_ids)); + } return std::make_unique(options_, std::move(macro_registry), std::move(library_ids)); } @@ -1903,9 +1911,19 @@ absl::StatusOr EnrichedParse( absl::StatusOr EnrichedParse( const cel::Source& source, const cel::MacroRegistry& registry, const ParserOptions& options) { + ParsedExpr parsed_expr; + if (options.enable_pratt_parser) { + CEL_ASSIGN_OR_RETURN( + std::unique_ptr ast, + cel::parser_internal::PrattParseImpl(source, registry, options)); + CEL_RETURN_IF_ERROR(cel::ast_internal::ExprToProto( + ast->root_expr(), parsed_expr.mutable_expr())); + CEL_RETURN_IF_ERROR(cel::ast_internal::SourceInfoToProto( + ast->source_info(), parsed_expr.mutable_source_info())); + return VerboseParsedExpr(std::move(parsed_expr), EnrichedSourceInfo()); + } CEL_ASSIGN_OR_RETURN(ParseResult parse_result, ParseImpl(source, registry, options)); - ParsedExpr parsed_expr; CEL_RETURN_IF_ERROR(cel::ast_internal::ExprToProto( parse_result.expr, parsed_expr.mutable_expr())); diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 357a83a79..9ea6e7832 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include @@ -56,8 +56,16 @@ using ::testing::Not; struct TestInfo { TestInfo(const std::string& I, const std::string& P, const std::string& E = "", const std::string& L = "", - const std::string& R = "", const std::string& M = "") - : I(I), P(P), E(E), L(L), R(R), M(M) {} + const std::string& R = "", const std::string& M = "", + const std::string& P_PRATT = "", const std::string& E_PRATT = "") + : I(I), + P(P), + E(E), + L(L), + R(R), + M(M), + P_PRATT(P_PRATT), + E_PRATT(E_PRATT) {} // I contains the input expression to be parsed. std::string I; @@ -77,6 +85,12 @@ struct TestInfo { // M contains the expected macro call output of hte expression tree. std::string M; + + // P_PRATT contains alternative adorned AST string when using pratt parser. + std::string P_PRATT; + + // E_PRATT contains alternative error output when using pratt parser. + std::string E_PRATT; }; std::vector test_cases = { @@ -130,6 +144,12 @@ std::vector test_cases = { "{\n" " foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" " bar^#6:Expr.Ident#:\"xyz\"^#7:string#^#5:Expr.CreateStruct.Entry#\n" + "}^#1:Expr.CreateStruct#", + "", "", "", "", + // PRATT PARSER AST + "{\n" + " foo^#2:Expr.Ident#:5^#4:int64#^#3:Expr.CreateStruct.Entry#,\n" + " bar^#5:Expr.Ident#:\"xyz\"^#7:string#^#6:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"a > 5 && a < 10", "_&&_(\n" @@ -160,6 +180,11 @@ std::vector test_cases = { "NUM_FLOAT, " "NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n | {\n" + " | .^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:2: expected '}'\n" + " | {\n" " | .^"}, // test cases from Go @@ -353,6 +378,12 @@ std::vector test_cases = { "{\n" " a^#3:Expr.Ident#:b^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry#,\n" " c^#6:Expr.Ident#:d^#7:Expr.Ident#^#5:Expr.CreateStruct.Entry#\n" + "}^#1:Expr.CreateStruct#", + "", "", "", "", + // PRATT PARSER AST + "{\n" + " a^#2:Expr.Ident#:b^#4:Expr.Ident#^#3:Expr.CreateStruct.Entry#,\n" + " c^#5:Expr.Ident#:d^#7:Expr.Ident#^#6:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"[]", "[]^#1:Expr.CreateList#"}, {"[a]", @@ -417,14 +448,27 @@ std::vector test_cases = { " | ....^\n" "ERROR: :1:7: Syntax error: extraneous input 'b' expecting \n" " | *@a | b\n" - " | ......^"}, + " | ......^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: unexpected token\n" + " | *@a | b\n" + " | ^\n" + "ERROR: :1:2: unexpected character\n" + " | *@a | b\n" + " | .^"}, {"a | b", "", "ERROR: :1:3: Syntax error: token recognition error at: '| '\n" " | a | b\n" " | ..^\n" "ERROR: :1:5: Syntax error: extraneous input 'b' expecting \n" " | a | b\n" - " | ....^"}, + " | ....^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: unexpected single '|', expected '||'\n" + " | a | b\n" + " | ..^"}, {"?", "", "ERROR: :1:1: Syntax error: mismatched input '?' expecting " "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " @@ -432,13 +476,23 @@ std::vector test_cases = { "ERROR: :1:2: Syntax error: mismatched input '' expecting " "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n | ?\n | .^\n" - "ERROR: :4294967295:0: <> parsetree"}, + "ERROR: :4294967295:0: <> parsetree", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: unexpected token\n" + " | ?\n" + " | ^"}, {"t{>C}", "", "ERROR: :1:3: Syntax error: extraneous input '>' expecting {'}', " "',', '\\u003F', IDENTIFIER, ESC_IDENTIFIER}\n | t{>C}\n | ..^\nERROR: " ":1:5: " "Syntax error: " - "mismatched input '}' expecting ':'\n | t{>C}\n | ....^"}, + "mismatched input '}' expecting ':'\n | t{>C}\n | ....^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: expected struct field name\n" + " | t{>C}\n" + " | ..^"}, // Macro tests {"has(m.f)", "m^#2:Expr.Ident#.f~test-only~^#4:Expr.Select#", "", @@ -584,6 +638,12 @@ std::vector test_cases = { "{\n" " 1^#3:int64#:2u^#4:uint64#^#2:Expr.CreateStruct.Entry#,\n" " 2^#6:int64#:3u^#7:uint64#^#5:Expr.CreateStruct.Entry#\n" + "}^#1:Expr.CreateStruct#", + "", "", "", "", + // PRATT PARSER AST + "{\n" + " 1^#2:int64#:2u^#4:uint64#^#3:Expr.CreateStruct.Entry#,\n" + " 2^#5:int64#:3u^#7:uint64#^#6:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"TestAllTypes{single_int32: 1, single_int64: 2}", "TestAllTypes{\n" @@ -593,6 +653,11 @@ std::vector test_cases = { {"TestAllTypes(){single_int32: 1, single_int64: 2}", "", "ERROR: :1:15: Syntax error: mismatched input '{' expecting \n" " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" + " | ..............^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:15: unexpected token after expression\n" + " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" " | ..............^"}, {"size(x) == x.size()", "_==_(\n" @@ -610,12 +675,22 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | 1 + $\n" - " | .....^"}, + " | .....^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:5: unexpected character\n" + " | 1 + $\n" + " | ....^"}, {"1 + 2\n" "3 +", "", "ERROR: :2:1: Syntax error: mismatched input '3' expecting \n" " | 3 +\n" + " | ^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :2:1: unexpected token after expression\n" + " | 3 +\n" " | ^"}, {"\"\\\"\"", "\"\\\"\"^#1:string#"}, {"[1,3,4][0]", @@ -634,24 +709,50 @@ std::vector test_cases = { {"[].all(.x, x)", "", "ERROR: :1:9: all() variable name must be a simple identifier\n" " | [].all(.x, x)\n" - " | ........^"}, + " | ........^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:8: all() variable name must be a simple identifier\n" + " | [].all(.x, x)\n" + " | .......^"}, {"[].exists(.x, x)", "", "ERROR: :1:12: exists() variable name must be a simple identifier\n" " | [].exists(.x, x)\n" - " | ...........^"}, + " | ...........^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:11: exists() variable name must be a simple identifier\n" + " | [].exists(.x, x)\n" + " | ..........^"}, {"[].exists_one(.x, x)", "", "ERROR: :1:16: exists_one() variable name must be a simple " "identifier\n" " | [].exists_one(.x, x)\n" - " | ...............^"}, + " | ...............^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:15: exists_one() variable name must be a simple " + "identifier\n" + " | [].exists_one(.x, x)\n" + " | ..............^"}, {"[].map(.x, x, x)", "", "ERROR: :1:9: map() variable name must be a simple identifier\n" " | [].map(.x, x, x)\n" - " | ........^"}, + " | ........^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:8: map() variable name must be a simple identifier\n" + " | [].map(.x, x, x)\n" + " | .......^"}, {"[].filter(.x, x)", "", "ERROR: :1:12: filter() variable name must be a simple identifier\n" " | [].filter(.x, x)\n" - " | ...........^"}, + " | ...........^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:11: filter() variable name must be a simple identifier\n" + " | [].filter(.x, x)\n" + " | ..........^"}, {"x[\"a\"].single_int32 == 23", "_==_(\n" " _[_](\n" @@ -707,6 +808,15 @@ std::vector test_cases = { {"---a", "-_(\n" " a^#2:Expr.Ident#\n" + ")^#1:Expr.Call#", + "", "", "", "", + // PRATT PARSER AST + "-_(\n" + " -_(\n" + " -_(\n" + " a^#4:Expr.Ident#\n" + " )^#3:Expr.Call#\n" + " )^#2:Expr.Call#\n" ")^#1:Expr.Call#"}, {"1 + +", "", "ERROR: :1:5: Syntax error: mismatched input '+' expecting {'[', " @@ -721,7 +831,12 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | 1 + +\n" - " | .....^"}, + " | .....^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:5: unexpected token\n" + " | 1 + +\n" + " | ....^"}, {"\"abc\" + \"def\"", "_+_(\n" " \"abc\"^#1:string#,\n" @@ -731,6 +846,11 @@ std::vector test_cases = { "ERROR: :1:10: Syntax error: no viable alternative at input " "'.\"a\"'\n" " | {\"a\": 1}.\"a\"\n" + " | .........^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:10: expected identifier after '.'\n" + " | {\"a\": 1}.\"a\"\n" " | .........^"}, {"\"\\xC3\\XBF\"", "\"ÿ\"^#1:string#"}, {"\"\\303\\277\"", "\"ÿ\"^#1:string#"}, @@ -750,7 +870,13 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | \"\\xFh\"\n" - " | ......^"}, + " | ......^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: Invalid string literal: Illegal escape sequence: Hex " + "escape must be followed by 2 hex digits but saw: \\xFh\n" + " | \"\\xFh\"\n" + " | ^"}, {"\"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\"", "", "ERROR: :1:1: Syntax error: token recognition error at: " "'\"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>'\n" @@ -764,7 +890,13 @@ std::vector test_cases = { " '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | \"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\"\n" - " | ..........................................^"}, + " | ..........................................^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: Invalid string literal: Illegal escape sequence: " + "\\>\n" + " | \"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\"\n" + " | ^"}, {"'😁' in ['😁', '😑', '😦']", "@in(\n" " \"😁\"^#1:string#,\n" @@ -814,7 +946,15 @@ std::vector test_cases = { " | .........^\n" "ERROR: :2:11: Syntax error: no viable alternative at input '.'\n" " | && in.😁\n" - " | ..........^"}, + " | ..........^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :2:7: unexpected token\n" + " | && in.😁\n" + " | ......^\n" + "ERROR: :2:10: unexpected character\n" + " | && in.😁\n" + " | .........^"}, {"as", "", "ERROR: :1:1: reserved identifier: as\n" " | as\n" @@ -862,7 +1002,12 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | in\n" - " | ..^"}, + " | ..^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: unexpected token\n" + " | in\n" + " | ^"}, {"let", "", "ERROR: :1:1: reserved identifier: let\n" " | let\n" @@ -907,6 +1052,17 @@ std::vector test_cases = { " | ...................^\n" "ERROR: :1:26: reserved identifier: var\n" " | [1, 2, 3].map(var, var * var)\n" + " | .........................^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:15: reserved identifier: var\n" + " | [1, 2, 3].map(var, var * var)\n" + " | ..............^\n" + "ERROR: :1:20: reserved identifier: var\n" + " | [1, 2, 3].map(var, var * var)\n" + " | ...................^\n" + "ERROR: :1:26: reserved identifier: var\n" + " | [1, 2, 3].map(var, var * var)\n" " | .........................^"}, {"[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" @@ -929,14 +1085,17 @@ std::vector test_cases = { "", "", }, - { - "[\n\t\r[\n\t\r[\n\t\r]\n\t\r]\n\t\r", - "", // parse output not validated as it is too large. - "ERROR: :6:3: Syntax error: mismatched input '' expecting " - "{']', ','}\n" - " | \r\n" - " | ..^", - }, + {"[\n\t\r[\n\t\r[\n\t\r]\n\t\r]\n\t\r", "", + // parse output not validated as it is too large. + "ERROR: :6:3: Syntax error: mismatched input '' expecting " + "{']', ','}\n" + " | \r\n" + " | ..^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :6:3: expected ']'\n" + " | \r\n" + " | ..^"}, // Identifier quoting syntax tests. {"a.`b`", "a^#1:Expr.Ident#.b^#2:Expr.Select#"}, @@ -977,21 +1136,36 @@ std::vector test_cases = { " | ..^\n" "ERROR: :1:7: Syntax error: token recognition error at: '`'\n" " | a.`b c`\n" - " | ......^"}, + " | ......^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`b c`\n" + " | ..^"}, {"a.`@foo`", "", "ERROR: :1:3: Syntax error: token recognition error at: '`@'\n" " | a.`@foo`\n" " | ..^\n" "ERROR: :1:8: Syntax error: token recognition error at: '`'\n" " | a.`@foo`\n" - " | .......^"}, + " | .......^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`@foo`\n" + " | ..^"}, {"a.`$foo`", "", "ERROR: :1:3: Syntax error: token recognition error at: '`$'\n" " | a.`$foo`\n" " | ..^\n" "ERROR: :1:8: Syntax error: token recognition error at: '`'\n" " | a.`$foo`\n" - " | .......^"}, + " | .......^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:3: unexpected quoted identifier\n" + " | a.`$foo`\n" + " | ..^"}, {"`a.b`", "", "ERROR: :1:1: Syntax error: mismatched input '`a.b`' expecting " "{'[', '{', " @@ -999,6 +1173,11 @@ std::vector test_cases = { "NUM_UINT, STRING, " "BYTES, IDENTIFIER}\n" " | `a.b`\n" + " | ^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: unexpected quoted identifier\n" + " | `a.b`\n" " | ^"}, {"`a.b`()", "", "ERROR: :1:1: Syntax error: extraneous input '`a.b`' expecting " @@ -1010,11 +1189,21 @@ std::vector test_cases = { "'{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM" "_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | `a.b`()\n" - " | ......^"}, + " | ......^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:1: unexpected quoted identifier\n" + " | `a.b`()\n" + " | ^"}, {"foo.`a.b`()", "", "ERROR: :1:10: Syntax error: mismatched input '(' expecting \n" " | foo.`a.b`()\n" - " | .........^"}, + " | .........^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:5: unexpected quoted identifier\n" + " | foo.`a.b`()\n" + " | ....^"}, // Macro calls tests {"x.filter(y, y.filter(z, z > 0))", @@ -1290,7 +1479,12 @@ std::vector test_cases = { {"{?'key': value}", "{\n " "?\"key\"^#3:string#:value^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry#\n}^#" - "1:Expr.CreateStruct#"}, + "1:Expr.CreateStruct#", + "", "", "", "", + // PRATT PARSER AST + "{\n" + " ?\"key\"^#2:string#:value^#4:Expr.Ident#^#3:Expr.CreateStruct.Entry#\n" + "}^#1:Expr.CreateStruct#"}, {"[?a, ?b]", "[\n ?a^#2:Expr.Ident#,\n ?b^#3:Expr.Ident#\n]^#1:Expr.CreateList#"}, {"[?a[?b]]", @@ -1489,46 +1683,59 @@ std::string ConvertMacroCallsToString( return result.substr(0, result.size() - 3); } -class ExpressionTest : public testing::TestWithParam {}; +class ExpressionTest + : public testing::TestWithParam> { + protected: + ExpressionTest() { options_.enable_pratt_parser = std::get<1>(GetParam()); } + + ParserOptions options_; +}; TEST_P(ExpressionTest, Parse) { - const TestInfo& test_info = GetParam(); - ParserOptions options; + const TestInfo& test_info = std::get<0>(GetParam()); if (!test_info.M.empty()) { - options.add_macro_calls = true; + options_.add_macro_calls = true; } - options.enable_optional_syntax = true; - options.enable_quoted_identifiers = true; + options_.enable_optional_syntax = true; + options_.enable_quoted_identifiers = true; std::vector macros = Macro::AllMacros(); macros.push_back(cel::OptMapMacro()); macros.push_back(cel::OptFlatMapMacro()); - auto result = EnrichedParse(test_info.I, macros, "", options); + auto result = EnrichedParse(test_info.I, macros, "", options_); if (test_info.E.empty()) { ASSERT_THAT(result, IsOk()); } else { EXPECT_THAT(result, Not(IsOk())); - EXPECT_EQ(test_info.E, result.status().message()); + if (options_.enable_pratt_parser && !test_info.E_PRATT.empty()) { + EXPECT_EQ(test_info.E_PRATT, result.status().message()); + } else { + EXPECT_EQ(test_info.E, result.status().message()); + } } if (!test_info.P.empty()) { KindAndIdAdorner kind_and_id_adorner; ExprPrinter w(kind_and_id_adorner); std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); - EXPECT_EQ(test_info.P, adorned_string) - << result->parsed_expr().ShortDebugString(); + if (options_.enable_pratt_parser && !test_info.P_PRATT.empty()) { + EXPECT_EQ(test_info.P_PRATT, adorned_string) + << result->parsed_expr().ShortDebugString(); + } else { + EXPECT_EQ(test_info.P, adorned_string) + << result->parsed_expr().ShortDebugString(); + } } - if (!test_info.L.empty()) { + if (!options_.enable_pratt_parser && !test_info.L.empty()) { LocationAdorner location_adorner(result->parsed_expr().source_info()); ExprPrinter w(location_adorner); std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); EXPECT_EQ(test_info.L, adorned_string) << result->parsed_expr().ShortDebugString(); - ; } - if (!test_info.R.empty()) { + if (!options_.enable_pratt_parser && !test_info.R.empty()) { EXPECT_EQ(test_info.R, ConvertEnrichedSourceInfoToString( result->enriched_source_info())); } @@ -1537,7 +1744,6 @@ TEST_P(ExpressionTest, Parse) { EXPECT_EQ(test_info.M, ConvertMacroCallsToString( result.value().parsed_expr().source_info())) << result->parsed_expr().ShortDebugString(); - ; } } @@ -1564,7 +1770,14 @@ TEST(ExpressionTest, CompositeExpressionOffsets) { EXPECT_EQ(msg_offsets.at(1), std::make_pair(0, 8)); } -TEST(ExpressionTest, TsanOom) { +class ExpressionImplTest : public testing::TestWithParam { + protected: + ExpressionImplTest() { options_.enable_pratt_parser = GetParam(); } + + ParserOptions options_; +}; + +TEST_P(ExpressionImplTest, TsanOom) { Parse( "[[a([[???[a[[??[a([[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" @@ -1580,7 +1793,8 @@ TEST(ExpressionTest, TsanOom) { "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[" "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[???[" - "a([[????") + "a([[????", + "", options_) .IgnoreError(); } @@ -1595,18 +1809,16 @@ TEST_P(ExpressionTest, ErrorRecoveryLimits) { "'..'\n | ......\n | .^"); } -TEST(ExpressionTest, ExpressionSizeLimit) { - ParserOptions options; - options.expression_size_codepoint_limit = 10; - auto result = Parse("...............", "", options); +TEST_P(ExpressionImplTest, ExpressionSizeLimit) { + options_.expression_size_codepoint_limit = 10; + auto result = Parse("...............", "", options_); EXPECT_THAT(result, Not(IsOk())); EXPECT_EQ( result.status().message(), "expression size exceeds codepoint limit. input size: 15, limit: 10"); } -TEST(ExpressionTest, RecursionDepthLongArgList) { - ParserOptions options; +TEST_P(ExpressionImplTest, RecursionDepthLongArgList) { // The particular number here is an implementation detail: the underlying // visitor will recurse up to 8 times before branching to the create list or // const steps. The call graph looks something like: @@ -1615,9 +1827,9 @@ TEST(ExpressionTest, RecursionDepthLongArgList) { // ->visitCreateList->visit[arg]->visitExpr... // The expected max depth for create list with an arbitrary number of elements // is 15. - options.max_recursion_depth = 16; + options_.max_recursion_depth = 16; - EXPECT_THAT(Parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]", "", options), IsOk()); + EXPECT_THAT(Parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]", "", options_), IsOk()); } TEST(ExpressionTest, RecursionDepthExceeded) { @@ -1634,10 +1846,9 @@ TEST(ExpressionTest, RecursionDepthExceeded) { HasSubstr("Exceeded max recursion depth of 6 when parsing.")); } -TEST(ExpressionTest, DisableQuotedIdentifiers) { - ParserOptions options; - options.enable_quoted_identifiers = false; - auto result = Parse("foo.`bar`", "", options); +TEST_P(ExpressionImplTest, DisableQuotedIdentifiers) { + options_.enable_quoted_identifiers = false; + auto result = Parse("foo.`bar`", "", options_); EXPECT_THAT(result, Not(IsOk())); EXPECT_THAT(result.status().message(), @@ -1646,11 +1857,10 @@ TEST(ExpressionTest, DisableQuotedIdentifiers) { " | ....^")); } -TEST(ExpressionTest, DisableStandardMacros) { - ParserOptions options; - options.disable_standard_macros = true; +TEST_P(ExpressionImplTest, DisableStandardMacros) { + options_.disable_standard_macros = true; - auto result = Parse("has(foo.bar)", "", options); + auto result = Parse("has(foo.bar)", "", options_); ASSERT_THAT(result, IsOk()); KindAndIdAdorner kind_and_id_adorner; @@ -1671,8 +1881,15 @@ TEST(ExpressionTest, RecursionDepthIgnoresParentheses) { EXPECT_THAT(result, IsOk()); } -TEST(NewParserBuilderTest, Defaults) { - auto builder = cel::NewParserBuilder(); +class NewParserBuilderTest : public testing::TestWithParam { + protected: + NewParserBuilderTest() { options_.enable_pratt_parser = GetParam(); } + + ParserOptions options_; +}; + +TEST_P(NewParserBuilderTest, Defaults) { + auto builder = cel::NewParserBuilder(options_); ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); ASSERT_OK_AND_ASSIGN(auto source, @@ -1682,8 +1899,8 @@ TEST(NewParserBuilderTest, Defaults) { EXPECT_FALSE(ast->IsChecked()); } -TEST(NewParserBuilderTest, CustomMacros) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, CustomMacros) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().disable_standard_macros = true; ASSERT_THAT(builder->AddMacro(cel::HasMacro()), IsOk()); ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); @@ -1705,8 +1922,8 @@ TEST(NewParserBuilderTest, CustomMacros) { ")^#9:Expr.Call#"); } -TEST(NewParserBuilderTest, StandardMacrosNotAddedWithStdlib) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, StandardMacrosNotAddedWithStdlib) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().disable_standard_macros = false; // Add a fake stdlib to check that we don't try to add the standard macros // again. Emulates what happens when we add support for subsetting stdlib by @@ -1735,15 +1952,15 @@ TEST(NewParserBuilderTest, StandardMacrosNotAddedWithStdlib) { ")^#9:Expr.Call#"); } -TEST(NewParserBuilderTest, ForwardsOptions) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, ForwardsOptions) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().enable_optional_syntax = true; ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("a.?b")); ASSERT_OK_AND_ASSIGN(auto ast, parser->Parse(*source)); EXPECT_FALSE(ast->IsChecked()); - builder = cel::NewParserBuilder(); + builder = cel::NewParserBuilder(options_); builder->GetOptions().enable_optional_syntax = false; ASSERT_OK_AND_ASSIGN(parser, std::move(*builder).Build()); ASSERT_OK_AND_ASSIGN(source, cel::NewSource("a.?b")); @@ -1751,8 +1968,8 @@ TEST(NewParserBuilderTest, ForwardsOptions) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST(NewParserBuilderTest, ToBuilderCopiesConfig) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, ToBuilderCopiesConfig) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().enable_optional_syntax = true; builder->GetOptions().disable_standard_macros = true; ASSERT_THAT(builder->AddLibrary({"custom_lib", @@ -1773,8 +1990,8 @@ TEST(NewParserBuilderTest, ToBuilderCopiesConfig) { EXPECT_FALSE(ast->IsChecked()); } -TEST(NewParserBuilderTest, ToBuilderHandlesStdlibAndOptionalByLibrary) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, ToBuilderHandlesStdlibAndOptionalByLibrary) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().disable_standard_macros = true; builder->GetOptions().enable_optional_syntax = false; @@ -1809,8 +2026,8 @@ TEST(NewParserBuilderTest, ToBuilderHandlesStdlibAndOptionalByLibrary) { ")^#1:Expr.Call#"); } -TEST(NewParserBuilderTest, ToBuilderPreservesStdlibAndOptionalFromOptions) { - auto builder = cel::NewParserBuilder(); +TEST_P(NewParserBuilderTest, ToBuilderPreservesStdlibAndOptionalFromOptions) { + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().disable_standard_macros = false; builder->GetOptions().enable_optional_syntax = true; @@ -1832,11 +2049,19 @@ struct VariadicLogicalOperatorsTestCase { }; class VariadicLogicalOperatorsTest - : public testing::TestWithParam {}; + : public testing::TestWithParam< + std::tuple> { + protected: + VariadicLogicalOperatorsTest() { + options_.enable_pratt_parser = std::get<1>(GetParam()); + } + + ParserOptions options_; +}; TEST_P(VariadicLogicalOperatorsTest, Parse) { - const auto& test_case = GetParam(); - auto builder = cel::NewParserBuilder(); + const auto& test_case = std::get<0>(GetParam()); + auto builder = cel::NewParserBuilder(options_); builder->GetOptions().enable_variadic_logical_operators = true; ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(test_case.input)); @@ -1848,39 +2073,56 @@ TEST_P(VariadicLogicalOperatorsTest, Parse) { EXPECT_EQ(adorned_string, test_case.expected_adorned_string); } +std::string VariadicLogicalOperatorsTestName( + const testing::TestParamInfo< + std::tuple>& test_info) { + bool enable_pratt = std::get<1>(test_info.param); + return absl::StrCat(test_info.index, "_", enable_pratt ? "Pratt" : "Legacy"); +} + INSTANTIATE_TEST_SUITE_P( VariadicLogicalOperators, VariadicLogicalOperatorsTest, - testing::Values( - VariadicLogicalOperatorsTestCase{ - .input = "a && b && c && d", - .expected_adorned_string = "_&&_(\n" - " a^#1:Expr.Ident#,\n" - " b^#2:Expr.Ident#,\n" - " c^#4:Expr.Ident#,\n" - " d^#6:Expr.Ident#\n" - ")^#3:Expr.Call#"}, - VariadicLogicalOperatorsTestCase{ - .input = "a || b || c || d", - .expected_adorned_string = "_||_(\n" - " a^#1:Expr.Ident#,\n" - " b^#2:Expr.Ident#,\n" - " c^#4:Expr.Ident#,\n" - " d^#6:Expr.Ident#\n" - ")^#3:Expr.Call#"}, - VariadicLogicalOperatorsTestCase{ - .input = "a && b && (c || d || e)", - .expected_adorned_string = "_&&_(\n" - " a^#1:Expr.Ident#,\n" - " b^#2:Expr.Ident#,\n" - " _||_(\n" - " c^#4:Expr.Ident#,\n" - " d^#5:Expr.Ident#,\n" - " e^#7:Expr.Ident#\n" - " )^#6:Expr.Call#\n" - ")^#3:Expr.Call#"})); - -TEST(ParserTest, ParseFailurePopulatesIssues) { - auto builder = cel::NewParserBuilder(); + testing::Combine( + testing::Values( + VariadicLogicalOperatorsTestCase{ + .input = "a && b && c && d", + .expected_adorned_string = "_&&_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " c^#4:Expr.Ident#,\n" + " d^#6:Expr.Ident#\n" + ")^#3:Expr.Call#"}, + VariadicLogicalOperatorsTestCase{ + .input = "a || b || c || d", + .expected_adorned_string = "_||_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " c^#4:Expr.Ident#,\n" + " d^#6:Expr.Ident#\n" + ")^#3:Expr.Call#"}, + VariadicLogicalOperatorsTestCase{ + .input = "a && b && (c || d || e)", + .expected_adorned_string = "_&&_(\n" + " a^#1:Expr.Ident#,\n" + " b^#2:Expr.Ident#,\n" + " _||_(\n" + " c^#4:Expr.Ident#,\n" + " d^#5:Expr.Ident#,\n" + " e^#7:Expr.Ident#\n" + " )^#6:Expr.Call#\n" + ")^#3:Expr.Call#"}), + testing::Bool()), + VariadicLogicalOperatorsTestName); + +class ParserTest : public testing::TestWithParam { + protected: + ParserTest() { options_.enable_pratt_parser = GetParam(); } + + ParserOptions options_; +}; + +TEST_P(ParserTest, ParseFailurePopulatesIssues) { + auto builder = cel::NewParserBuilder(options_); ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("a +", "test.cel")); @@ -1898,15 +2140,33 @@ TEST(ParserTest, ParseFailurePopulatesIssues) { EXPECT_EQ(issues[0].location().column, 3); } -std::string TestName(const testing::TestParamInfo& test_info) { - std::string name = absl::StrCat(test_info.index, "-", test_info.param.I); +std::string ExpressionTestName( + const testing::TestParamInfo>& test_info) { + const TestInfo& info = std::get<0>(test_info.param); + bool enable_pratt = std::get<1>(test_info.param); + std::string name = absl::StrCat(test_info.index, "_", + enable_pratt ? "Pratt_" : "Legacy_", info.I); absl::c_replace_if(name, [](char c) { return !absl::ascii_isalnum(c); }, '_'); return name; - return name; } INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionTest, - testing::ValuesIn(test_cases), TestName); + testing::Combine(testing::ValuesIn(test_cases), + testing::Bool()), + ExpressionTestName); + +std::string ParserImplTestName(const testing::TestParamInfo& info) { + return info.param ? "Pratt" : "Legacy"; +} + +INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionImplTest, testing::Bool(), + ParserImplTestName); + +INSTANTIATE_TEST_SUITE_P(CelParserTest, NewParserBuilderTest, testing::Bool(), + ParserImplTestName); + +INSTANTIATE_TEST_SUITE_P(CelParserTest, ParserTest, testing::Bool(), + ParserImplTestName); } // namespace } // namespace google::api::expr::parser From 7a1c706760e94b2509c9ae8ce112c92e948c629e Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 27 Jul 2026 12:57:04 -0700 Subject: [PATCH 143/350] Add support for typed has operations. PiperOrigin-RevId: 954783233 --- eval/eval/select_step.cc | 66 +++++++++- eval/tests/BUILD | 1 - eval/tests/benchmark_test.cc | 192 ++++++++++++++++++++++------ eval/tests/modern_benchmark_test.cc | 169 ++++++++++++++++++------ 4 files changed, 348 insertions(+), 80 deletions(-) diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 8078f99f8..0b31c3c13 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -471,6 +471,62 @@ absl::Status ProtoSelectStep::EvaluateLegacyMessageGetField( &frame->value_stack().Peek()); } +class ProtoHasStep : public SelectStep { + public: + ProtoHasStep(StringValue value, int64_t expr_id, + bool enable_wrapper_type_null_unboxing, + bool enable_optional_types, const google::protobuf::Descriptor* descriptor, + const google::protobuf::FieldDescriptor* field_descriptor) + : SelectStep(std::move(value), /*test_field_presence=*/true, expr_id, + enable_wrapper_type_null_unboxing, enable_optional_types), + descriptor_(descriptor), + field_descriptor_(field_descriptor) { + ABSL_DCHECK(descriptor_ != nullptr); + ABSL_DCHECK(field_descriptor_ != nullptr); + } + + absl::Status Evaluate(ExecutionFrame* frame) const override { + if (!frame->value_stack().HasEnough(1)) { + return absl::InternalError( + "No arguments supplied for Select-type expression"); + } + + const Value& arg = frame->value_stack().Peek(); + if (auto unwrapped = arg.AsParsedMessage(); + unwrapped.has_value() && unwrapped->GetDescriptor() == descriptor_) { + return EvaluateHas(frame, *unwrapped); + } else if (const google::protobuf::Message* legacy_message = + cel::interop_internal::GetLegacyMessage(arg); + legacy_message != nullptr && + legacy_message->GetDescriptor() == descriptor_) { + cel::ParsedMessageValue parsed_message = + cel::UnsafeParsedMessageValue(legacy_message); + return EvaluateHas(frame, parsed_message); + } + // If we get an unexpected value type, fall back to the generic + // implementation. + return SelectStep::Evaluate(frame); + } + + private: + absl::Status EvaluateHas(ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const; + + const google::protobuf::Descriptor* descriptor_; + const google::protobuf::FieldDescriptor* field_descriptor_; +}; + +absl::Status ProtoHasStep::EvaluateHas( + ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const { + if (CheckAttributeTrail(field_, frame)) { + return absl::OkStatus(); + } + frame->value_stack().Peek() = + BoolValue{parsed_message.HasField(field_descriptor_)}; + return absl::OkStatus(); +} + } // namespace std::unique_ptr CreateDirectSelectStep( @@ -496,10 +552,10 @@ absl::StatusOr> CreateTypedSelectStep( cel::StringValue field, cel::StructType resolved_operand_type, cel::StructTypeField resolved_field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { - if (!resolved_operand_type.IsMessage() || test_only) { + if (!resolved_operand_type.IsMessage()) { // The specialization only supports messages. Fallback to the generic // implementation for other types. - // TODO(uncreated-issue/89): support has() for messages. + // TODO(uncreated-issue/89): support optional select and chaining. return CreateSelectStep(std::move(field), test_only, expr_id, enable_wrapper_type_null_unboxing, enable_optional_types); @@ -511,6 +567,12 @@ absl::StatusOr> CreateTypedSelectStep( const google::protobuf::FieldDescriptor* field_descriptor = resolved_field.GetMessage().descriptor(); + if (test_only) { + return std::make_unique( + std::move(field), expr_id, enable_wrapper_type_null_unboxing, + enable_optional_types, descriptor, field_descriptor); + } + return std::make_unique( std::move(field), expr_id, enable_wrapper_type_null_unboxing, enable_optional_types, descriptor, field_descriptor); diff --git a/eval/tests/BUILD b/eval/tests/BUILD index c5a7a7062..e4dacea3f 100644 --- a/eval/tests/BUILD +++ b/eval/tests/BUILD @@ -85,7 +85,6 @@ cc_test( "//internal:benchmark", "//internal:testing", "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", "//parser", "//parser:macro_registry", "//runtime", diff --git a/eval/tests/benchmark_test.cc b/eval/tests/benchmark_test.cc index abeac608a..ff38ed2af 100644 --- a/eval/tests/benchmark_test.cc +++ b/eval/tests/benchmark_test.cc @@ -592,16 +592,34 @@ BENCHMARK(BM_HasMap); void BM_HasProto(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, - parser::Parse("has(request.path) && !has(request.ip)")); + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN( + auto validation_result, + compiler->Compile("has(request.path) && !has(request.ip)")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; request.set_path(kPath); request.set_token(kToken); @@ -620,17 +638,34 @@ BENCHMARK(BM_HasProto); void BM_HasProtoMap(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, - parser::Parse("has(request.headers.create_time) && " - "!has(request.headers.update_time)")); + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, + compiler->Compile("has(request.headers.create_time) && " + "!has(request.headers.update_time)")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; request.mutable_headers()->insert({"create_time", "2021-01-01"}); activation.InsertValue("request", @@ -648,17 +683,34 @@ BENCHMARK(BM_HasProtoMap); void BM_ReadProtoMap(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( request.headers.create_time == "2021-01-01" )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; request.mutable_headers()->insert({"create_time", "2021-01-01"}); activation.InsertValue("request", @@ -676,17 +728,34 @@ BENCHMARK(BM_ReadProtoMap); void BM_NestedProtoFieldRead(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( !request.a.b.c.d.e )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; request.mutable_a()->mutable_b()->mutable_c()->mutable_d()->set_e(false); activation.InsertValue("request", @@ -704,17 +773,34 @@ BENCHMARK(BM_NestedProtoFieldRead); void BM_NestedProtoFieldReadDefaults(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( !request.a.b.c.d.e )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); - auto reg_status = RegisterBuiltinFunctions(builder->GetRegistry(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), + IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; RequestContext request; activation.InsertValue("request", CelProtoWrapper::CreateMessage(&request, &arena)); @@ -731,18 +817,35 @@ BENCHMARK(BM_NestedProtoFieldReadDefaults); void BM_ProtoStructAccess(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", + cel::MessageType(AttributeContext::Request::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( has(request.auth.claims.iss) && request.auth.claims.iss == 'accounts.google.com' )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; AttributeContext::Request request; auto* auth = request.mutable_auth(); (*auth->mutable_claims()->mutable_fields())["iss"].set_string_value( @@ -762,18 +865,35 @@ BENCHMARK(BM_ProtoStructAccess); void BM_ProtoListAccess(benchmark::State& state) { google::protobuf::Arena arena; - Activation activation; - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(R"cel( + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + cel::NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(compiler_builder->AddLibrary(cel::StandardCompilerLibrary()), + IsOk()); + ASSERT_THAT( + compiler_builder->GetCheckerBuilder().AddVariable(cel::MakeVariableDecl( + "request", + cel::MessageType(AttributeContext::Request::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, compiler_builder->Build()); + + ASSERT_OK_AND_ASSIGN(auto validation_result, compiler->Compile(R"cel( "//.../accessLevels/MY_LEVEL_4" in request.auth.access_levels )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + cel::expr::CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*ast, &checked_expr), IsOk()); + InterpreterOptions options = GetOptions(arena); auto builder = CreateCelExpressionBuilder(options); ASSERT_THAT(RegisterBuiltinFunctions(builder->GetRegistry(), options), IsOk()); - ASSERT_OK_AND_ASSIGN(auto cel_expr, - builder->CreateExpression(&parsed_expr.expr(), nullptr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&checked_expr)); + Activation activation; AttributeContext::Request request; auto* auth = request.mutable_auth(); auth->add_access_levels("//.../accessLevels/MY_LEVEL_0"); diff --git a/eval/tests/modern_benchmark_test.cc b/eval/tests/modern_benchmark_test.cc index 28daed981..8a41b094f 100644 --- a/eval/tests/modern_benchmark_test.cc +++ b/eval/tests/modern_benchmark_test.cc @@ -702,12 +702,25 @@ BENCHMARK(BM_HasMap); void BM_HasProto(benchmark::State& state) { RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, - Parse("has(request.path) && !has(request.ip)")); - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN( + ValidationResult validation_result, + compiler->Compile("has(request.path) && !has(request.ip)")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -729,13 +742,25 @@ BENCHMARK(BM_HasProto); void BM_HasProtoMap(benchmark::State& state) { RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, - Parse("has(request.headers.create_time) && " - "!has(request.headers.update_time)")); - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile("has(request.headers.create_time) && " + "!has(request.headers.update_time)")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); + + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -755,15 +780,27 @@ void BM_HasProtoMap(benchmark::State& state) { BENCHMARK(BM_HasProtoMap); void BM_ReadProtoMap(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( request.headers.create_time == "2021-01-01" )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -783,15 +820,27 @@ void BM_ReadProtoMap(benchmark::State& state) { BENCHMARK(BM_ReadProtoMap); void BM_NestedProtoFieldRead(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !request.a.b.c.d.e )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -811,15 +860,27 @@ void BM_NestedProtoFieldRead(benchmark::State& state) { BENCHMARK(BM_NestedProtoFieldRead); void BM_NestedProtoFieldReadDefaults(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", cel::MessageType(RequestContext::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( !request.a.b.c.d.e )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -838,15 +899,28 @@ void BM_NestedProtoFieldReadDefaults(benchmark::State& state) { BENCHMARK(BM_NestedProtoFieldReadDefaults); void BM_ProtoStructAccess(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", + cel::MessageType(AttributeContext::Request::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( has(request.auth.claims.iss) && request.auth.claims.iss == 'accounts.google.com' )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; @@ -868,15 +942,28 @@ void BM_ProtoStructAccess(benchmark::State& state) { BENCHMARK(BM_ProtoStructAccess); void BM_ProtoListAccess(benchmark::State& state) { - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse(R"cel( + RuntimeOptions options = GetOptions(); + auto runtime = + StandardRuntimeOrDie(options, google::protobuf::DescriptorPool::generated_pool()); + + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(google::protobuf::DescriptorPool::generated_pool())); + ASSERT_THAT(builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(builder->GetCheckerBuilder().AddVariable(MakeVariableDecl( + "request", + cel::MessageType(AttributeContext::Request::descriptor()))), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult validation_result, + compiler->Compile(R"cel( "//.../accessLevels/MY_LEVEL_4" in request.auth.access_levels )cel")); + ASSERT_TRUE(validation_result.IsValid()); + ASSERT_OK_AND_ASSIGN(auto ast, validation_result.ReleaseAst()); - RuntimeOptions options = GetOptions(); - auto runtime = StandardRuntimeOrDie(options); - - ASSERT_OK_AND_ASSIGN(auto cel_expr, ProtobufRuntimeAdapter::CreateProgram( - *runtime, parsed_expr)); + ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); google::protobuf::Arena arena; Activation activation; From db06cd9449be81aa0ba76dc47424e12622611a0f Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 27 Jul 2026 15:10:44 -0700 Subject: [PATCH 144/350] Test fix: use hermetic descriptor pool PiperOrigin-RevId: 954852824 --- eval/public/structs/BUILD | 2 ++ .../structs/protobuf_descriptor_type_provider_test.cc | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 2c7d7ed16..4e4d5481c 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -371,6 +371,8 @@ cc_test( "//eval/public/testing:matchers", "//extensions/protobuf:memory_manager", "//internal:testing", + "//internal:testing_descriptor_pool", + "//internal:testing_message_factory", "@com_google_absl//absl/status:status_matchers", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/eval/public/structs/protobuf_descriptor_type_provider_test.cc b/eval/public/structs/protobuf_descriptor_type_provider_test.cc index aabdc38c4..1d07fd657 100644 --- a/eval/public/structs/protobuf_descriptor_type_provider_test.cc +++ b/eval/public/structs/protobuf_descriptor_type_provider_test.cc @@ -24,6 +24,8 @@ #include "eval/public/testing/matchers.h" #include "extensions/protobuf/memory_manager.h" #include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -101,8 +103,8 @@ TEST(ProtobufDescriptorProvider, NotFound) { TEST(ProtobufDescriptorProvider, FindType) { ProtobufDescriptorProvider provider( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory()); ASSERT_OK_AND_ASSIGN(std::optional wrapper_type, provider.FindType("google.protobuf.Int64Value")); ASSERT_TRUE(wrapper_type.has_value()); From 02b617c63bc9d09314ac1cf37569c75f3c36cdb0 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 27 Jul 2026 15:27:23 -0700 Subject: [PATCH 145/350] Add limit for expression nodes in parsed ASTs. This is mainly to prevent misbehaving macros from generating an AST that is much larger than expected (not directly proportional to source length). PiperOrigin-RevId: 954860770 --- parser/internal/options.h | 1 + parser/internal/pratt_parser_worker.cc | 10 ++ parser/internal/pratt_parser_worker.h | 8 +- parser/options.h | 8 ++ parser/parser.cc | 122 +++++++++++++++---------- parser/parser_test.cc | 49 ++++++++++ 6 files changed, 151 insertions(+), 47 deletions(-) diff --git a/parser/internal/options.h b/parser/internal/options.h index 4066d2619..cca9670a5 100644 --- a/parser/internal/options.h +++ b/parser/internal/options.h @@ -21,6 +21,7 @@ inline constexpr int kDefaultErrorRecoveryLimit = 12; inline constexpr int kDefaultMaxRecursionDepth = 32; inline constexpr int kExpressionSizeCodepointLimit = 100'000; inline constexpr int kDefaultErrorRecoveryTokenLookaheadLimit = 512; +inline constexpr int kDefaultExpressionNodeLimit = 100'000; inline constexpr bool kDefaultAddMacroCalls = false; } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index c502796f5..f9efca03c 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -192,14 +192,24 @@ void ParserWorker::SynchronizeOnDelimiter() { NextToken(); } } + int64_t ParserWorker::NextId(int32_t position) { int64_t id = next_id_++; + if (id > options_.expression_node_limit) { + ReportError(position, "expression node limit exceeded"); + } if (position >= 0) { positions_.insert({id, position}); } return id; } +int64_t ParserWorker::NextId() { return NextId(-1); } + +bool ParserWorker::NodeLimitExceeded() { + return next_id_ > options_.expression_node_limit; +} + int64_t ParserWorker::CopyId(int64_t id) { if (id == 0) { return 0; diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index b022ae9ee..2e8534e78 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -76,7 +76,8 @@ class ParserWorker { // ID and Position tracking int64_t NextId(int32_t position); int64_t NextId(const Token& token) { return NextId(token.start); } - int64_t NextId() { return next_id_++; } + int64_t NextId(); + bool NodeLimitExceeded(); int64_t CopyId(int64_t id); void EraseId(int64_t id); @@ -1050,6 +1051,11 @@ std::optional PrattParserWorker::TryExpandMacro( if (!expander) { return std::nullopt; } + if (NodeLimitExceeded()) { + ReportError(expr_id, + "could not expand macro: expression node limit exceeded"); + return std::nullopt; + } std::vector macro_args; ExprNode macro_target; diff --git a/parser/options.h b/parser/options.h index eca41810a..eb9d2eeea 100644 --- a/parser/options.h +++ b/parser/options.h @@ -42,6 +42,14 @@ struct ParserOptions final { int error_recovery_token_lookahead_limit = ::cel::parser_internal::kDefaultErrorRecoveryTokenLookaheadLimit; + // Limit on the number of expression nodes in the abstract syntax tree for the + // expression. This prevents cases where macro expansion results in an AST + // that is larger than expected from the source expression. Once exceeded, + // the parser will record an error and stop expanding macros but continue + // parsing to report other errors. + int expression_node_limit = + ::cel::parser_internal::kDefaultExpressionNodeLimit; + // Add macro calls to macro_calls list in source_info. bool add_macro_calls = ::cel::parser_internal::kDefaultAddMacroCalls; diff --git a/parser/parser.cc b/parser/parser.cc index 6a66ffc9a..9bc471fdf 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -165,8 +165,9 @@ SourceRange SourceRangeFromParserRuleContext( class ParserMacroExprFactory final : public MacroExprFactory { public: - explicit ParserMacroExprFactory(const cel::Source& source) - : source_(source) {} + explicit ParserMacroExprFactory(const cel::Source& source, + int expression_node_limit) + : source_(source), expression_node_limit_(expression_node_limit) {} void BeginMacro(SourceRange macro_position) { macro_position_ = macro_position; @@ -203,12 +204,18 @@ class ParserMacroExprFactory final : public MacroExprFactory { int64_t NextId(const SourceRange& range) { auto id = expr_id_++; + if (id > expression_node_limit_ && !node_limit_exceeded_) { + node_limit_exceeded_ = true; + ReportError(range, "expression node limit exceeded"); + } if (range.begin != -1 || range.end != -1) { positions_.insert(std::pair{id, range}); } return id; } + bool is_node_limit_exceeded() const { return node_limit_exceeded_; } + bool HasErrors() const { return error_count_ != 0; } std::vector CollectIssues() { @@ -409,6 +416,8 @@ class ParserMacroExprFactory final : public MacroExprFactory { std::vector errors_; size_t error_count_ = 0; const Source& source_; + int expression_node_limit_; + bool node_limit_exceeded_ = false; SourceRange macro_position_; }; @@ -623,13 +632,14 @@ class ParserVisitor final : public CelBaseVisitor, public antlr4::BaseErrorListener { public: ParserVisitor(const cel::Source& source, int max_recursion_depth, + int max_expression_node_count, const cel::MacroRegistry& macro_registry, bool add_macro_calls = false, bool enable_optional_syntax = false, bool enable_quoted_identifiers = false, bool enable_variadic_logical_operators = false) : source_(source), - factory_(source_), + factory_(source_, max_expression_node_count), macro_registry_(macro_registry), recursion_depth_(0), max_recursion_depth_(max_recursion_depth), @@ -1227,6 +1237,8 @@ std::vector ParserVisitor::visitList( if (!enable_optional_syntax_ && expr_ctx->opt != nullptr) { factory_.ReportError(SourceRangeFromParserRuleContext(ctx), "unsupported syntax '?'"); + // Still generate an ID to detect node limit exceeded. + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); rv.push_back(factory_.NewListElement(factory_.NewUnspecified(0), false)); continue; } @@ -1298,6 +1310,9 @@ std::vector ParserVisitor::visitEntries( if (!enable_optional_syntax_ && ctx->keys[i]->opt) { factory_.ReportError(SourceRangeFromParserRuleContext(ctx), "unsupported syntax '?'"); + // Still generate an ID to detect node limit exceeded. + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); res.push_back(factory_.NewMapEntry(0, factory_.NewUnspecified(0), factory_.NewUnspecified(0), false)); continue; @@ -1461,29 +1476,34 @@ std::vector ParserVisitor::CollectIssues() { Expr ParserVisitor::GlobalCallOrMacroImpl(int64_t expr_id, absl::string_view function, std::vector args) { - if (auto macro = macro_registry_.FindMacro(function, args.size(), false); - macro) { - std::vector macro_args; - if (add_macro_calls_) { - macro_args.reserve(args.size()); - for (const auto& arg : args) { - macro_args.push_back(factory_.BuildMacroCallArg(arg)); - } + auto macro = macro_registry_.FindMacro(function, args.size(), false); + if (!macro) { + return factory_.NewCall(expr_id, function, std::move(args)); + } + if (factory_.is_node_limit_exceeded()) { + return factory_.ReportError( + factory_.GetSourceRange(expr_id), + "could not expand macro: expression node limit exceeded"); + } + std::vector macro_args; + if (add_macro_calls_) { + macro_args.reserve(args.size()); + for (const auto& arg : args) { + macro_args.push_back(factory_.BuildMacroCallArg(arg)); } - factory_.BeginMacro(factory_.GetSourceRange(expr_id)); - auto expr = macro->Expand(factory_, std::nullopt, absl::MakeSpan(args)); - factory_.EndMacro(); - if (expr) { - if (add_macro_calls_) { - factory_.AddMacroCall(expr->id(), function, std::nullopt, - std::move(macro_args)); - } - // We did not end up using `expr_id`. Delete metadata. - factory_.EraseId(expr_id); - return std::move(*expr); + } + factory_.BeginMacro(factory_.GetSourceRange(expr_id)); + auto expr = macro->Expand(factory_, std::nullopt, absl::MakeSpan(args)); + factory_.EndMacro(); + if (expr) { + if (add_macro_calls_) { + factory_.AddMacroCall(expr->id(), function, std::nullopt, + std::move(macro_args)); } + // We did not end up using `expr_id`. Delete metadata. + factory_.EraseId(expr_id); + return std::move(*expr); } - return factory_.NewCall(expr_id, function, std::move(args)); } @@ -1491,30 +1511,39 @@ Expr ParserVisitor::ReceiverCallOrMacroImpl(int64_t expr_id, absl::string_view function, Expr target, std::vector args) { - if (auto macro = macro_registry_.FindMacro(function, args.size(), true); - macro) { - Expr macro_target; - std::vector macro_args; - if (add_macro_calls_) { - macro_args.reserve(args.size()); - macro_target = factory_.BuildMacroCallArg(target); - for (const auto& arg : args) { - macro_args.push_back(factory_.BuildMacroCallArg(arg)); - } + auto macro = macro_registry_.FindMacro(function, args.size(), true); + if (!macro) { + return factory_.NewMemberCall(expr_id, function, std::move(target), + std::move(args)); + } + if (factory_.is_node_limit_exceeded()) { + return factory_.ReportError( + factory_.GetSourceRange(expr_id), + "could not expand macro: expression node limit exceeded"); + } + + Expr macro_target; + std::vector macro_args; + if (add_macro_calls_) { + macro_args.reserve(args.size()); + macro_target = factory_.BuildMacroCallArg(target); + for (const auto& arg : args) { + macro_args.push_back(factory_.BuildMacroCallArg(arg)); } - factory_.BeginMacro(factory_.GetSourceRange(expr_id)); - auto expr = macro->Expand(factory_, std::ref(target), absl::MakeSpan(args)); - factory_.EndMacro(); - if (expr) { - if (add_macro_calls_) { - factory_.AddMacroCall(expr->id(), function, std::move(macro_target), - std::move(macro_args)); - } - // We did not end up using `expr_id`. Delete metadata. - factory_.EraseId(expr_id); - return std::move(*expr); + } + factory_.BeginMacro(factory_.GetSourceRange(expr_id)); + auto expr = macro->Expand(factory_, std::ref(target), absl::MakeSpan(args)); + factory_.EndMacro(); + if (expr) { + if (add_macro_calls_) { + factory_.AddMacroCall(expr->id(), function, std::move(macro_target), + std::move(macro_args)); } + // We did not end up using `expr_id`. Delete metadata. + factory_.EraseId(expr_id); + return std::move(*expr); } + return factory_.NewMemberCall(expr_id, function, std::move(target), std::move(args)); } @@ -1677,8 +1706,9 @@ absl::StatusOr ParseImpl( CelParser parser(&tokens); ExprRecursionListener listener(options.max_recursion_depth); ParserVisitor visitor( - source, options.max_recursion_depth, registry, options.add_macro_calls, - options.enable_optional_syntax, options.enable_quoted_identifiers, + source, options.max_recursion_depth, options.expression_node_limit, + registry, options.add_macro_calls, options.enable_optional_syntax, + options.enable_quoted_identifiers, options.enable_variadic_logical_operators); lexer.removeErrorListeners(); diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 9ea6e7832..2f0aae3e3 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -2140,6 +2140,55 @@ TEST_P(ParserTest, ParseFailurePopulatesIssues) { EXPECT_EQ(issues[0].location().column, 3); } +TEST_P(ParserTest, ExpressionNodeLimitExceeded) { + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().expression_node_limit = 2; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("a + b + c", "test.cel")); + std::vector issues; + auto ast_result = parser->Parse(*source, &issues); + EXPECT_THAT(ast_result, Not(IsOk())); + ASSERT_THAT(issues, testing::Not(testing::IsEmpty())); + EXPECT_THAT(ast_result.status().message(), + HasSubstr("expression node limit exceeded")); + EXPECT_THAT(issues[0].message(), HasSubstr("expression node limit exceeded")); +} + +TEST_P(ParserTest, MacroExpansionNodeLimitExceeded) { + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().expression_node_limit = 5; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + ASSERT_OK_AND_ASSIGN( + auto source, cel::NewSource("[1, 2, 3, 4, 5].map(x, x * 2)", "test.cel")); + std::vector issues; + auto ast_result = parser->Parse(*source, &issues); + EXPECT_THAT(ast_result, Not(IsOk())); + ASSERT_THAT(issues, testing::Not(testing::IsEmpty())); + EXPECT_THAT(ast_result.status().message(), + HasSubstr("expression node limit exceeded")); + EXPECT_THAT( + issues, + testing::Contains(testing::Property( + &cel::ParseIssue::message, + HasSubstr( + "could not expand macro: expression node limit exceeded")))); +} + +TEST_P(ParserTest, MacroExpansionNodeLimitNotExceeded) { + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().expression_node_limit = 100; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + ASSERT_OK_AND_ASSIGN( + auto source, cel::NewSource("[1, 2, 3, 4, 5].map(x, x * 2)", "test.cel")); + std::vector issues; + auto ast_result = parser->Parse(*source, &issues); + ASSERT_THAT(ast_result, IsOk()); + EXPECT_THAT(issues, testing::IsEmpty()); +} + std::string ExpressionTestName( const testing::TestParamInfo>& test_info) { const TestInfo& info = std::get<0>(test_info.param); From 0160bfd04926028f4815a6c28c4c4ac66e5f2437 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 27 Jul 2026 16:46:45 -0700 Subject: [PATCH 146/350] Add missing #undef 'IN' for windows PiperOrigin-RevId: 954901437 --- parser/internal/pratt_parser_worker.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index f9efca03c..b37578297 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -48,7 +48,10 @@ const BinaryOpInfo kEqualEqual = {3, CelOperator::EQUALS, false, TokenType::kEqualEqual}; const BinaryOpInfo kExclamationEqual = {3, CelOperator::NOT_EQUALS, false, TokenType::kExclamationEqual}; +#pragma push_macro("IN") +#undef IN const BinaryOpInfo kIn = {3, CelOperator::IN, false, TokenType::kIn}; +#pragma pop_macro("IN") const BinaryOpInfo kPlus = {4, CelOperator::ADD, false, TokenType::kPlus}; const BinaryOpInfo kMinus = {4, CelOperator::SUBTRACT, false, TokenType::kMinus}; From 21620e279204b9dd87fc741f057070f7ad9e30a5 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 27 Jul 2026 17:34:13 -0700 Subject: [PATCH 147/350] Avoid duplicating complex expressions in optional macros cross ref: https://github.com/cel-expr/cel-go/pull/1387 PiperOrigin-RevId: 954922600 --- compiler/optional_test.cc | 102 +++++++++++++++++++++++--------------- parser/macro.cc | 92 ++++++++++++++++++++++++++++------ 2 files changed, 139 insertions(+), 55 deletions(-) diff --git a/compiler/optional_test.cc b/compiler/optional_test.cc index 699c69f76..f5091b9fc 100644 --- a/compiler/optional_test.cc +++ b/compiler/optional_test.cc @@ -111,11 +111,66 @@ optional.of( TestCase{ .expr = "optional.of('foo').optMap(x, x)", .expected_ast = R"( -_?_:_( +__comprehension__( + // Variable + #unused, + // Target + []~list(dyn), + // Accumulator + @target, + // Init optional.of( "foo"~string - )~optional_type(string)^optional_of.hasValue()~bool^optional_hasValue, + )~optional_type(string)^optional_of, + // LoopCondition + false~bool, + // LoopStep + @target~optional_type(string)^@target, + // Result + _?_:_( + @target~optional_type(string)^@target.hasValue()~bool^optional_hasValue, + optional.of( + __comprehension__( + // Variable + #unused, + // Target + []~list(dyn), + // Accumulator + x, + // Init + @target~optional_type(string)^@target.value()~string^optional_value, + // LoopCondition + false~bool, + // LoopStep + x~string^x, + // Result + x~string^x)~string + )~optional_type(string)^optional_of, + optional.none()~optional_type(string)^optional_none + )~optional_type(string)^conditional)~optional_type(string) +)", + }, + TestCase{ + .expr = "optional.of('foo').optFlatMap(x, optional.of(x))", + .expected_ast = R"( +__comprehension__( + // Variable + #unused, + // Target + []~list(dyn), + // Accumulator + @target, + // Init optional.of( + "foo"~string + )~optional_type(string)^optional_of, + // LoopCondition + false~bool, + // LoopStep + @target~optional_type(string)^@target, + // Result + _?_:_( + @target~optional_type(string)^@target.hasValue()~bool^optional_hasValue, __comprehension__( // Variable #unused, @@ -124,48 +179,17 @@ _?_:_( // Accumulator x, // Init - optional.of( - "foo"~string - )~optional_type(string)^optional_of.value()~string^optional_value, + @target~optional_type(string)^@target.value()~string^optional_value, // LoopCondition false~bool, // LoopStep x~string^x, // Result - x~string^x)~string - )~optional_type(string)^optional_of, - optional.none()~optional_type(string)^optional_none -)~optional_type(string)^conditional -)", - }, - TestCase{ - .expr = "optional.of('foo').optFlatMap(x, optional.of(x))", - .expected_ast = R"( -_?_:_( - optional.of( - "foo"~string - )~optional_type(string)^optional_of.hasValue()~bool^optional_hasValue, - __comprehension__( - // Variable - #unused, - // Target - []~list(dyn), - // Accumulator - x, - // Init - optional.of( - "foo"~string - )~optional_type(string)^optional_of.value()~string^optional_value, - // LoopCondition - false~bool, - // LoopStep - x~string^x, - // Result - optional.of( - x~string^x - )~optional_type(string)^optional_of)~optional_type(string), - optional.none()~optional_type(string)^optional_none -)~optional_type(string)^conditional + optional.of( + x~string^x + )~optional_type(string)^optional_of)~optional_type(string), + optional.none()~optional_type(string)^optional_none + )~optional_type(string)^conditional)~optional_type(string) )", }, TestCase{ diff --git a/parser/macro.cc b/parser/macro.cc index 815b07401..057787e8d 100644 --- a/parser/macro.cc +++ b/parser/macro.cc @@ -39,6 +39,8 @@ namespace cel { namespace { +constexpr absl::string_view kOptionalMapVar = "@target"; + using google::api::expr::common::CelOperator; bool IsSimpleIdentifier(const Expr& expr) { @@ -315,20 +317,50 @@ absl::optional ExpandOptMapMacro(MacroExprFactory& factory, Expr& target, } auto var_name = args[0].ident_expr().name(); - auto target_copy = factory.Copy(target); - std::vector call_args; - call_args.reserve(3); - call_args.push_back(factory.NewMemberCall("hasValue", std::move(target))); + if (target.has_ident_expr()) { + auto target_copy = factory.Copy(target); + std::vector call_args; + call_args.reserve(3); + call_args.push_back(factory.NewMemberCall("hasValue", std::move(target))); + auto iter_range = factory.NewList(); + auto accu_init = factory.NewMemberCall("value", std::move(target_copy)); + auto condition = factory.NewBoolConst(false); + auto fold = factory.NewComprehension( + "#unused", std::move(iter_range), std::move(var_name), + std::move(accu_init), std::move(condition), std::move(args[0]), + std::move(args[1])); + call_args.push_back(factory.NewCall("optional.of", std::move(fold))); + call_args.push_back(factory.NewCall("optional.none")); + return factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + } + + // If the target is complex, use an internal bind expression to avoid + // repeating it and blowing up the AST in the expansion + auto tmp = factory.NewIdent(kOptionalMapVar); + auto tmp_copy = factory.Copy(tmp); + auto iter_range = factory.NewList(); - auto accu_init = factory.NewMemberCall("value", std::move(target_copy)); + auto accu_init = factory.NewMemberCall("value", std::move(tmp_copy)); auto condition = factory.NewBoolConst(false); + auto loop_step = std::move(args[0]); auto fold = factory.NewComprehension( "#unused", std::move(iter_range), std::move(var_name), - std::move(accu_init), std::move(condition), std::move(args[0]), + std::move(accu_init), std::move(condition), std::move(loop_step), std::move(args[1])); + std::vector call_args; + call_args.reserve(3); + call_args.push_back(factory.NewMemberCall("hasValue", std::move(tmp))); call_args.push_back(factory.NewCall("optional.of", std::move(fold))); call_args.push_back(factory.NewCall("optional.none")); - return factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + auto result = factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + + iter_range = factory.NewList(); + accu_init = std::move(target); + condition = factory.NewBoolConst(false); + loop_step = factory.NewIdent(kOptionalMapVar); + return factory.NewComprehension( + "#unused", std::move(iter_range), kOptionalMapVar, std::move(accu_init), + std::move(condition), loop_step, std::move(result)); } Macro MakeOptMapMacro() { @@ -354,19 +386,47 @@ absl::optional ExpandOptFlatMapMacro(MacroExprFactory& factory, } auto var_name = args[0].ident_expr().name(); - auto target_copy = factory.Copy(target); - std::vector call_args; - call_args.reserve(3); - call_args.push_back(factory.NewMemberCall("hasValue", std::move(target))); + if (target.has_ident_expr()) { + auto target_copy = factory.Copy(target); + std::vector call_args; + call_args.reserve(3); + call_args.push_back(factory.NewMemberCall("hasValue", std::move(target))); + auto iter_range = factory.NewList(); + auto accu_init = factory.NewMemberCall("value", std::move(target_copy)); + auto condition = factory.NewBoolConst(false); + call_args.push_back(factory.NewComprehension( + "#unused", std::move(iter_range), std::move(var_name), + std::move(accu_init), std::move(condition), std::move(args[0]), + std::move(args[1]))); + call_args.push_back(factory.NewCall("optional.none")); + return factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + } + + auto tmp = factory.NewIdent(kOptionalMapVar); + auto tmp_copy = factory.Copy(tmp); + auto iter_range = factory.NewList(); - auto accu_init = factory.NewMemberCall("value", std::move(target_copy)); + auto accu_init = factory.NewMemberCall("value", std::move(tmp_copy)); auto condition = factory.NewBoolConst(false); - call_args.push_back(factory.NewComprehension( + auto loop_step = std::move(args[0]); + auto inner = factory.NewComprehension( "#unused", std::move(iter_range), std::move(var_name), - std::move(accu_init), std::move(condition), std::move(args[0]), - std::move(args[1]))); + std::move(accu_init), std::move(condition), std::move(loop_step), + std::move(args[1])); + std::vector call_args; + call_args.reserve(3); + call_args.push_back(factory.NewMemberCall("hasValue", std::move(tmp))); + call_args.push_back(std::move(inner)); call_args.push_back(factory.NewCall("optional.none")); - return factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + auto result = factory.NewCall(CelOperator::CONDITIONAL, std::move(call_args)); + + iter_range = factory.NewList(); + accu_init = std::move(target); + condition = factory.NewBoolConst(false); + loop_step = factory.NewIdent(kOptionalMapVar); + return factory.NewComprehension( + "#unused", std::move(iter_range), kOptionalMapVar, std::move(accu_init), + std::move(condition), loop_step, std::move(result)); } Macro MakeOptFlatMapMacro() { From e25ca104899c8a886c867d52b771f04d62d684f0 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 27 Jul 2026 20:25:46 -0700 Subject: [PATCH 148/350] std::string_view->absl::string_view PiperOrigin-RevId: 954987806 --- parser/internal/pratt_parser_test.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 7b2570514..595aaf05f 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -67,7 +67,7 @@ std::string TestName(const testing::TestParamInfo& test_info) { } absl::StatusOr> Parse( - std::string_view expression, + absl::string_view expression, const cel::ParserOptions& options = cel::ParserOptions(), std::vector* issues = nullptr) { #if USE_PRATT_PARSER @@ -82,15 +82,15 @@ absl::StatusOr> Parse( } struct TestCase { - std::string_view source; - std::string_view expected_ast; + absl::string_view source; + absl::string_view expected_ast; bool enable_optional_syntax = false; bool enable_variadic_logical_operators = false; }; class PrattParserTest : public testing::TestWithParam {}; -std::string_view ConstantKind(const cel::Constant& c) { +absl::string_view ConstantKind(const cel::Constant& c) { switch (c.kind_case()) { case ConstantKindCase::kBool: return "bool"; @@ -111,7 +111,7 @@ std::string_view ConstantKind(const cel::Constant& c) { } } -std::string_view ExprKind(const cel::Expr& e) { +absl::string_view ExprKind(const cel::Expr& e) { switch (e.kind_case()) { case ExprKindCase::kConstant: // special cased, this doesn't appear. @@ -155,12 +155,12 @@ class KindAndIdAdorner : public cel::test::ExpressionAdorner { } }; -std::string Unindent(std::string_view multiline) { +std::string Unindent(absl::string_view multiline) { std::vector unindented_lines; int indent = -1; - for (std::string_view line : absl::StrSplit(multiline, '\n')) { + for (absl::string_view line : absl::StrSplit(multiline, '\n')) { std::size_t pos = line.find_first_not_of(" \t"); - if (pos == std::string_view::npos) continue; + if (pos == absl::string_view::npos) continue; if (indent == -1) indent = pos; unindented_lines.push_back(std::string(line.substr(indent))); } @@ -1095,8 +1095,8 @@ INSTANTIATE_TEST_SUITE_P(PrattParserTest, PrattParserTest, TestName); struct ErrorTestCase { - std::string_view source; - std::string_view expected_error; + absl::string_view source; + absl::string_view expected_error; bool enable_optional_syntax = false; bool enable_quoted_identifiers = false; }; @@ -1505,8 +1505,8 @@ TEST(ParserWorkerTest, GetTokenTextBoundsChecking) { } struct MacroTestCase { - std::string_view source; - std::string_view expected_ast; + absl::string_view source; + absl::string_view expected_ast; }; class PrattParserMacroTest : public testing::TestWithParam {}; From 7140d7f9e6edd38c14a386a3d68d1dabf574f3bd Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 27 Jul 2026 20:48:45 -0700 Subject: [PATCH 149/350] Opt-in to eager field lookup by default. PiperOrigin-RevId: 954995155 --- eval/public/cel_options.h | 2 +- runtime/runtime_options.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index 001990431..e72b33eb0 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -222,7 +222,7 @@ struct InterpreterOptions { // When enabled, the planner will attempt to use a more performant execution // path for field access when the type is known at plan time, instead of using // the generic field access implementation. - bool enable_typed_field_access = false; + bool enable_typed_field_access = true; }; // LINT.ThenChange(//depot/google3/runtime/runtime_options.h) diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 072daf26c..1624dfb1c 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -200,7 +200,7 @@ struct RuntimeOptions { // This is not recommended if the values bound to the activation are typically // not what the planner expected (e.g. a map that was declared as a proto or // a different message with matching field names). - bool enable_typed_field_access = false; + bool enable_typed_field_access = true; }; // LINT.ThenChange(//depot/google3/eval/public/cel_options.h) From 352093363d848978f63aa74c9aab40103a42430e Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 27 Jul 2026 22:06:49 -0700 Subject: [PATCH 150/350] No public description PiperOrigin-RevId: 955021319 --- internal/empty_descriptors.cc | 3 ++- internal/minimal_descriptors.cc | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/empty_descriptors.cc b/internal/empty_descriptors.cc index 05e3843a5..43fe90006 100644 --- a/internal/empty_descriptors.cc +++ b/internal/empty_descriptors.cc @@ -15,6 +15,7 @@ #include "internal/empty_descriptors.h" #include +#include #include "google/protobuf/descriptor.pb.h" #include "absl/base/attributes.h" @@ -39,7 +40,7 @@ const google::protobuf::DescriptorPool* absl_nonnull GetEmptyDescriptorPool() { static const google::protobuf::DescriptorPool* absl_nonnull const pool = []() { google::protobuf::FileDescriptorSet file_desc_set; ABSL_CHECK(file_desc_set.ParseFromArray( // Crash OK - kEmptyDescriptorSet, ABSL_ARRAYSIZE(kEmptyDescriptorSet))); + kEmptyDescriptorSet, std::size(kEmptyDescriptorSet))); auto* pool = new google::protobuf::DescriptorPool(); for (const auto& file_desc : file_desc_set.file()) { ABSL_CHECK(pool->BuildFile(file_desc) != nullptr); // Crash OK diff --git a/internal/minimal_descriptors.cc b/internal/minimal_descriptors.cc index f0b96e838..b8d56923d 100644 --- a/internal/minimal_descriptors.cc +++ b/internal/minimal_descriptors.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -43,7 +44,7 @@ const google::protobuf::FileDescriptorSet* GetMinimumFileDescriptorSet() { static google::protobuf::FileDescriptorSet* const file_desc_set = []() { google::protobuf::FileDescriptorSet* file_desc_set = new google::protobuf::FileDescriptorSet(); ABSL_CHECK(file_desc_set->ParseFromArray( // Crash OK - kMinimalDescriptorSet, ABSL_ARRAYSIZE(kMinimalDescriptorSet))); + kMinimalDescriptorSet, std::size(kMinimalDescriptorSet))); return file_desc_set; }(); return file_desc_set; From 7ee2cd38357b14adee650bdc5b51f24cd1e989d8 Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Tue, 28 Jul 2026 14:33:29 -0700 Subject: [PATCH 151/350] Moving Expr Printer to CEL common PiperOrigin-RevId: 955475683 --- common/BUILD | 33 +++++++++++++++++ {testutil => common}/expr_printer.cc | 6 ++-- {testutil => common}/expr_printer.h | 10 +++--- {testutil => common}/expr_printer_test.cc | 2 +- parser/BUILD | 4 +-- parser/internal/BUILD | 2 +- parser/internal/pratt_parser_test.cc | 6 ++-- parser/parser_test.cc | 8 ++--- policy/internal/BUILD | 2 +- .../internal/optimizer_expr_factory_test.cc | 11 +++--- testutil/BUILD | 35 +------------------ testutil/baseline_tests.cc | 2 +- 12 files changed, 60 insertions(+), 61 deletions(-) rename {testutil => common}/expr_printer.cc (98%) rename {testutil => common}/expr_printer.h (88%) rename {testutil => common}/expr_printer_test.cc (99%) diff --git a/common/BUILD b/common/BUILD index a6b57ab31..a1d4ac3bd 100644 --- a/common/BUILD +++ b/common/BUILD @@ -34,6 +34,39 @@ cc_library( ], ) +cc_library( + name = "expr_printer", + srcs = ["expr_printer.cc"], + hdrs = ["expr_printer.h"], + deps = [ + ":ast", + ":ast_proto", + ":constant", + ":expr", + "//internal:strings", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + ], +) + +cc_test( + name = "expr_printer_test", + srcs = ["expr_printer_test.cc"], + deps = [ + ":expr", + ":expr_printer", + "//internal:testing", + "//parser", + "//parser:options", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/strings", + ], +) + cc_test( name = "ast_test", srcs = ["ast_test.cc"], diff --git a/testutil/expr_printer.cc b/common/expr_printer.cc similarity index 98% rename from testutil/expr_printer.cc rename to common/expr_printer.cc index 40dea3c33..8a80777ea 100644 --- a/testutil/expr_printer.cc +++ b/common/expr_printer.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "testutil/expr_printer.h" +#include "common/expr_printer.h" #include #include @@ -29,7 +29,7 @@ #include "common/expr.h" #include "internal/strings.h" -namespace cel::test { +namespace cel { namespace { class EmptyAdornerImpl : public ExpressionAdorner { @@ -328,4 +328,4 @@ std::string ExprPrinter::Print(const Expr& expr) const { return w.Print(expr); } -} // namespace cel::test +} // namespace cel diff --git a/testutil/expr_printer.h b/common/expr_printer.h similarity index 88% rename from testutil/expr_printer.h rename to common/expr_printer.h index 6b0a8c161..9304276ef 100644 --- a/testutil/expr_printer.h +++ b/common/expr_printer.h @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_TESTUTIL_EXPR_PRINTER_H_ -#define THIRD_PARTY_CEL_CPP_TESTUTIL_EXPR_PRINTER_H_ +#ifndef THIRD_PARTY_CEL_CPP_COMMON_EXPR_PRINTER_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_EXPR_PRINTER_H_ #include #include "cel/expr/syntax.pb.h" #include "common/expr.h" -namespace cel::test { +namespace cel { // Interface for adding additional information to an expression during // printing. @@ -52,6 +52,6 @@ class ExprPrinter { const ExpressionAdorner& adorner_; }; -} // namespace cel::test +} // namespace cel -#endif // THIRD_PARTY_CEL_CPP_TESTUTIL_EXPR_PRINTER_H_ +#endif // THIRD_PARTY_CEL_CPP_COMMON_EXPR_PRINTER_H_ diff --git a/testutil/expr_printer_test.cc b/common/expr_printer_test.cc similarity index 99% rename from testutil/expr_printer_test.cc rename to common/expr_printer_test.cc index 9b1e7ca37..36646bfef 100644 --- a/testutil/expr_printer_test.cc +++ b/common/expr_printer_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "testutil/expr_printer.h" +#include "common/expr_printer.h" #include diff --git a/parser/BUILD b/parser/BUILD index 04a2c0516..c548967eb 100644 --- a/parser/BUILD +++ b/parser/BUILD @@ -186,9 +186,9 @@ cc_test( ":source_factory", "//common:constant", "//common:expr", + "//common:expr_printer", "//common:source", "//internal:testing", - "//testutil:expr_printer", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", @@ -210,10 +210,10 @@ cc_test( ":source_factory", "//common:constant", "//common:expr", + "//common:expr_printer", "//common:source", "//internal:benchmark", "//internal:testing", - "//testutil:expr_printer", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", diff --git a/parser/internal/BUILD b/parser/internal/BUILD index 8638fcc92..0cfb042de 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -169,6 +169,7 @@ cc_test( "//common:ast", "//common:constant", "//common:expr", + "//common:expr_printer", "//common:source", "//internal:status_macros", "//internal:testing", @@ -176,7 +177,6 @@ cc_test( "//parser:macro_expr_factory", "//parser:options", "//parser:parser_interface", - "//testutil:expr_printer", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 595aaf05f..539372e51 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -36,6 +36,7 @@ #include "common/ast.h" #include "common/constant.h" #include "common/expr.h" +#include "common/expr_printer.h" #include "common/source.h" #include "internal/status_macros.h" #include "internal/testing.h" @@ -45,7 +46,6 @@ #include "parser/macro_expr_factory.h" #include "parser/options.h" #include "parser/parser_interface.h" -#include "testutil/expr_printer.h" // Change to 0 to test with the ANTLR parser to check for differences. #define USE_PRATT_PARSER 1 @@ -135,7 +135,7 @@ absl::string_view ExprKind(const cel::Expr& e) { } } -class KindAndIdAdorner : public cel::test::ExpressionAdorner { +class KindAndIdAdorner : public cel::ExpressionAdorner { public: std::string Adorn(const cel::Expr& e) const override { if (e.has_const_expr()) { @@ -169,7 +169,7 @@ std::string Unindent(absl::string_view multiline) { MATCHER_P(AstIs, expected_ast, "") { KindAndIdAdorner kind_and_id_adorner; - test::ExprPrinter printer(kind_and_id_adorner); + cel::ExprPrinter printer(kind_and_id_adorner); std::string actual = Unindent(printer.Print(arg)); std::string expected = Unindent(expected_ast); if (actual == expected) { diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 2f0aae3e3..c2fd36112 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -32,13 +32,13 @@ #include "absl/types/optional.h" #include "common/constant.h" #include "common/expr.h" +#include "common/expr_printer.h" #include "common/source.h" #include "internal/testing.h" #include "parser/macro.h" #include "parser/options.h" #include "parser/parser_interface.h" #include "parser/source_factory.h" -#include "testutil/expr_printer.h" namespace google::api::expr::parser { @@ -48,7 +48,7 @@ using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::ConstantKindCase; using ::cel::ExprKindCase; -using ::cel::test::ExprPrinter; +using ::cel::ExprPrinter; using ::cel::expr::Expr; using ::testing::HasSubstr; using ::testing::Not; @@ -1556,7 +1556,7 @@ absl::string_view ExprKind(const cel::Expr& e) { } } -class KindAndIdAdorner : public cel::test::ExpressionAdorner { +class KindAndIdAdorner : public cel::ExpressionAdorner { public: // Use default source_info constructor to make source_info "optional". This // will prevent macro_calls lookups from interfering with adorning expressions @@ -1595,7 +1595,7 @@ class KindAndIdAdorner : public cel::test::ExpressionAdorner { const cel::expr::SourceInfo& source_info_; }; -class LocationAdorner : public cel::test::ExpressionAdorner { +class LocationAdorner : public cel::ExpressionAdorner { public: explicit LocationAdorner(const cel::expr::SourceInfo& source_info) : source_info_(source_info) {} diff --git a/policy/internal/BUILD b/policy/internal/BUILD index 98aeaeebb..bd4f7a4f3 100644 --- a/policy/internal/BUILD +++ b/policy/internal/BUILD @@ -49,6 +49,7 @@ cc_test( "//common:decl", "//common:expr", "//common:expr_factory", + "//common:expr_printer", "//common:source", "//common:type", "//compiler", @@ -57,7 +58,6 @@ cc_test( "//internal:status_macros", "//internal:testing", "//internal:testing_descriptor_pool", - "//testutil:expr_printer", "//tools:cel_unparser", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", diff --git a/policy/internal/optimizer_expr_factory_test.cc b/policy/internal/optimizer_expr_factory_test.cc index 1b14b5628..a85399411 100644 --- a/policy/internal/optimizer_expr_factory_test.cc +++ b/policy/internal/optimizer_expr_factory_test.cc @@ -29,6 +29,7 @@ #include "common/decl.h" #include "common/expr.h" #include "common/expr_factory.h" +#include "common/expr_printer.h" #include "common/source.h" #include "common/type.h" #include "compiler/compiler.h" @@ -37,7 +38,6 @@ #include "internal/status_macros.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" -#include "testutil/expr_printer.h" #include "tools/cel_unparser.h" namespace cel { @@ -347,7 +347,7 @@ TEST(OptimizerExprFactory, RecordReplacement) { EXPECT_EQ(arg.ident_expr().name(), "replacement"); } -class IdAdorner : public cel::test::ExpressionAdorner { +class IdAdorner : public cel::ExpressionAdorner { public: std::string Adorn(const cel::Expr& e) const override { return absl::StrCat("#", e.id()); @@ -398,9 +398,8 @@ TEST(OptimizerExprFactory, UnparseCopiedMacroCall) { factory.RecordReplacement(to_replace_id, copied_expr); // Test AST structure. - EXPECT_EQ( - cel::test::ExprPrinter(IdAdorner()).Print(factory.ast().root_expr()), - R"(__comprehension__( + EXPECT_THAT(cel::ExprPrinter(IdAdorner()).Print(factory.ast().root_expr()), + ::testing::StrEq(R"(__comprehension__( // Variable x, // Target @@ -452,7 +451,7 @@ TEST(OptimizerExprFactory, UnparseCopiedMacroCall) { ]#11 )#12, // Result - @result#13)#14)"); + @result#13)#14)")); // Check that the structure is compatible with unparser. cel::expr::ParsedExpr optimized_parsed; diff --git a/testutil/BUILD b/testutil/BUILD index 782c95ca6..3c1832652 100644 --- a/testutil/BUILD +++ b/testutil/BUILD @@ -20,39 +20,6 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( - name = "expr_printer", - srcs = ["expr_printer.cc"], - hdrs = ["expr_printer.h"], - deps = [ - "//common:ast", - "//common:ast_proto", - "//common:constant", - "//common:expr", - "//internal:strings", - "@com_google_absl//absl/base:no_destructor", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", - ], -) - -cc_test( - name = "expr_printer_test", - srcs = ["expr_printer_test.cc"], - deps = [ - ":expr_printer", - "//common:expr", - "//internal:testing", - "//parser", - "//parser:options", - "@com_google_absl//absl/base:no_destructor", - "@com_google_absl//absl/strings", - ], -) - cc_library( name = "util", testonly = True, @@ -88,9 +55,9 @@ cc_library( srcs = ["baseline_tests.cc"], hdrs = ["baseline_tests.h"], deps = [ - ":expr_printer", "//common:ast", "//common:expr", + "//common:expr_printer", "//extensions/protobuf:ast_converters", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", diff --git a/testutil/baseline_tests.cc b/testutil/baseline_tests.cc index 8ce43e63d..08db0827a 100644 --- a/testutil/baseline_tests.cc +++ b/testutil/baseline_tests.cc @@ -22,8 +22,8 @@ #include "absl/strings/str_join.h" #include "common/ast.h" #include "common/expr.h" +#include "common/expr_printer.h" #include "extensions/protobuf/ast_converters.h" -#include "testutil/expr_printer.h" namespace cel::test { namespace { From d96768d179cb121b0ad1ae6b360f04744e10414b Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 28 Jul 2026 15:11:22 -0700 Subject: [PATCH 152/350] Only report exceeding node count once. PiperOrigin-RevId: 955496224 --- parser/internal/pratt_parser_worker.cc | 7 +++---- parser/internal/pratt_parser_worker.h | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index b37578297..3f842ea80 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -198,8 +198,9 @@ void ParserWorker::SynchronizeOnDelimiter() { int64_t ParserWorker::NextId(int32_t position) { int64_t id = next_id_++; - if (id > options_.expression_node_limit) { + if (id > options_.expression_node_limit && !node_limit_exceeded_) { ReportError(position, "expression node limit exceeded"); + node_limit_exceeded_ = true; } if (position >= 0) { positions_.insert({id, position}); @@ -209,9 +210,7 @@ int64_t ParserWorker::NextId(int32_t position) { int64_t ParserWorker::NextId() { return NextId(-1); } -bool ParserWorker::NodeLimitExceeded() { - return next_id_ > options_.expression_node_limit; -} +bool ParserWorker::NodeLimitExceeded() { return node_limit_exceeded_; } int64_t ParserWorker::CopyId(int64_t id) { if (id == 0) { diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 2e8534e78..30fdce882 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -98,6 +98,7 @@ class ParserWorker { Token peek_token_; int recursion_depth_ = 0; int64_t next_id_ = 1; + bool node_limit_exceeded_ = false; absl::flat_hash_map positions_; std::vector* absl_nullable parse_issues_; int error_count_ = 0; From 54c0512418805e20491cae01b40452f4a2864d53 Mon Sep 17 00:00:00 2001 From: Antoine Pietri Date: Wed, 29 Jul 2026 13:06:46 -0700 Subject: [PATCH 153/350] Support cel::ActivationInterface in CelTestContext. Update `CelTestContext` and `CelActivationFactoryFn` to return `std::unique_ptr` instead of `cel::Activation` by value. Previously, `CelTestContext::CelActivationFactoryFn` was hardcoded to return `absl::StatusOr`. This overlooked the fact that `cel::Runtime::Evaluate` accepts `const cel::ActivationInterface&` and prevents custom or lazy activations that implement `cel::ActivationInterface` directly from being returned by the factory without object slicing or compilation errors. `CelTestContext` and `TestRunner` now accept `std::unique_ptr` from the factory function. PiperOrigin-RevId: 956045618 --- testing/testrunner/BUILD | 6 ++ testing/testrunner/cel_test_context.h | 9 ++- testing/testrunner/runner_lib.cc | 44 +++++++---- testing/testrunner/runner_lib_test.cc | 109 ++++++++++++++++++++++++-- 4 files changed, 143 insertions(+), 25 deletions(-) diff --git a/testing/testrunner/BUILD b/testing/testrunner/BUILD index b80167487..f01f892f3 100644 --- a/testing/testrunner/BUILD +++ b/testing/testrunner/BUILD @@ -18,6 +18,7 @@ cc_library( "//eval/public:cel_expression", "//runtime", "//runtime:activation", + "//runtime:activation_interface", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", @@ -51,6 +52,7 @@ cc_library( "//internal:testing_no_main", "//runtime", "//runtime:activation", + "//runtime:activation_interface", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", @@ -90,6 +92,7 @@ cc_test( ":cel_test_context", ":coverage_index", ":runner_lib", + "//base:attributes", "//checker:type_checker_builder", "//checker:validation_result", "//common:ast_proto", @@ -107,6 +110,8 @@ cc_test( "//internal:testing_descriptor_pool", "//runtime", "//runtime:activation", + "//runtime:activation_interface", + "//runtime:function_overload_reference", "//runtime:runtime_builder", "//runtime:standard_runtime_builder_factory", "@com_google_absl//absl/container:flat_hash_map", @@ -115,6 +120,7 @@ cc_test( "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/test:suite_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/testing/testrunner/cel_test_context.h b/testing/testrunner/cel_test_context.h index 0e0f21e28..988c37c66 100644 --- a/testing/testrunner/cel_test_context.h +++ b/testing/testrunner/cel_test_context.h @@ -29,7 +29,7 @@ #include "common/value.h" #include "compiler/compiler.h" #include "eval/public/cel_expression.h" -#include "runtime/activation.h" +#include "runtime/activation_interface.h" #include "runtime/runtime.h" #include "testing/testrunner/cel_expression_source.h" #include "cel/expr/conformance/test/suite.pb.h" @@ -40,9 +40,10 @@ namespace cel::test { // compiled CEL expressions. class CelTestContext { public: - using CelActivationFactoryFn = std::function( - const cel::expr::conformance::test::TestCase& test_case, - google::protobuf::Arena* arena)>; + using CelActivationFactoryFn = + std::function>( + const cel::expr::conformance::test::TestCase& test_case, + google::protobuf::Arena* arena)>; using AssertFn = std::function EvalWithModernBindings( const CheckedExpr& checked_expr, const CelTestContext& context, - const cel::Activation& activation, google::protobuf::Arena* arena) { + const cel::ActivationInterface& activation, google::protobuf::Arena* arena) { CEL_ASSIGN_OR_RETURN(std::unique_ptr program, Plan(checked_expr, context.runtime())); return program->Evaluate(arena, activation); @@ -205,25 +205,37 @@ absl::Status AddTestCaseBindingsToModernActivation( return absl::OkStatus(); } -absl::StatusOr GetActivation(const CelTestContext& context, - const TestCase& test_case, - google::protobuf::Arena* arena) { +absl::StatusOr> GetActivation( + const CelTestContext& context, const TestCase& test_case, + google::protobuf::Arena* arena) { if (context.activation_factory() != nullptr) { return context.activation_factory()(test_case, arena); } - return cel::Activation(); + return std::make_unique(); } -absl::StatusOr CreateModernActivationFromBindings( - const TestCase& test_case, const CelTestContext& context, - google::protobuf::Arena* arena) { - CEL_ASSIGN_OR_RETURN(cel::Activation activation, +absl::StatusOr> +CreateModernActivationFromBindings(const TestCase& test_case, + const CelTestContext& context, + google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN(std::unique_ptr activation, GetActivation(context, test_case, arena)); - CEL_RETURN_IF_ERROR( - AddCustomBindingsToModernActivation(context, activation, arena)); - CEL_RETURN_IF_ERROR(AddTestCaseBindingsToModernActivation(test_case, context, - activation, arena)); + const bool has_custom_bindings = + !context.custom_bindings().empty() || !test_case.input().empty(); + if (has_custom_bindings) { + auto* cel_activation = dynamic_cast(activation.get()); + if (cel_activation == nullptr) { + return absl::InvalidArgumentError( + "Custom bindings or test case input bindings cannot be combined with " + "a custom cel::ActivationInterface implementation returned by " + "activation_factory."); + } + CEL_RETURN_IF_ERROR( + AddCustomBindingsToModernActivation(context, *cel_activation, arena)); + CEL_RETURN_IF_ERROR(AddTestCaseBindingsToModernActivation( + test_case, context, *cel_activation, arena)); + } return activation; } @@ -362,9 +374,9 @@ absl::StatusOr TestRunner::EvalWithRuntime( const CheckedExpr& checked_expr, const TestCase& test_case, google::protobuf::Arena* arena) { CEL_ASSIGN_OR_RETURN( - cel::Activation activation, + std::unique_ptr activation, CreateModernActivationFromBindings(test_case, *test_context_, arena)); - return EvalWithModernBindings(checked_expr, *test_context_, activation, + return EvalWithModernBindings(checked_expr, *test_context_, *activation, arena); } diff --git a/testing/testrunner/runner_lib_test.cc b/testing/testrunner/runner_lib_test.cc index 804826b6c..3b0e3f586 100644 --- a/testing/testrunner/runner_lib_test.cc +++ b/testing/testrunner/runner_lib_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include "gtest/gtest-spi.h" #include "absl/container/flat_hash_map.h" @@ -25,6 +26,8 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" #include "checker/type_checker_builder.h" #include "checker/validation_result.h" #include "common/ast_proto.h" @@ -41,6 +44,8 @@ #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "runtime/activation.h" +#include "runtime/activation_interface.h" +#include "runtime/function_overload_reference.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" #include "runtime/standard_runtime_builder_factory.h" @@ -625,11 +630,11 @@ TEST(TestRunnerStandaloneTest, BasicTestWithActivationFactorySucceeds) { std::unique_ptr context = CelTestContext::CreateFromRuntime(std::move(runtime)); context->SetActivationFactory( - [](const TestCase& test_case, - google::protobuf::Arena* arena) -> absl::StatusOr { - cel::Activation activation; - activation.InsertOrAssignValue("x", cel::IntValue(10)); - activation.InsertOrAssignValue("y", cel::IntValue(5)); + [](const TestCase& test_case, google::protobuf::Arena* arena) + -> absl::StatusOr> { + auto activation = std::make_unique(); + activation->InsertOrAssignValue("x", cel::IntValue(10)); + activation->InsertOrAssignValue("y", cel::IntValue(5)); return activation; }); context->SetExpressionSource( @@ -652,6 +657,100 @@ TEST(TestRunnerStandaloneTest, BasicTestWithActivationFactorySucceeds) { EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); } +namespace { +class TestCustomActivation : public cel::ActivationInterface { + public: + absl::StatusOr FindVariable( + absl::string_view name, const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, google::protobuf::Arena* arena, + cel::Value* result) const override { + if (name == "x") { + *result = cel::IntValue(100); + return true; + } + if (name == "y") { + *result = cel::IntValue(200); + return true; + } + return false; + } + + std::vector FindFunctionOverloads( + absl::string_view name) const override { + return {}; + } + + absl::Span GetUnknownAttributes() + const override { + return {}; + } + + absl::Span GetMissingAttributes() + const override { + return {}; + } +}; +} // namespace + +TEST(TestRunnerStandaloneTest, CustomActivationInterfaceFactorySucceeds) { + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + DefaultCompiler().Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + std::unique_ptr context = + CelTestContext::CreateFromRuntime(std::move(runtime)); + context->SetActivationFactory( + [](const TestCase& test_case, google::protobuf::Arena* arena) + -> absl::StatusOr> { + return std::make_unique(); + }); + context->SetExpressionSource( + CelExpressionSource::FromCheckedExpr(std::move(checked_expr))); + + TestCase test_case = ParseTextProtoOrDie(R"pb( + output { result_value { int64_value: 300 } } + )pb"); + TestRunner test_runner(std::move(context)); + EXPECT_NO_FATAL_FAILURE(test_runner.RunTest(test_case)); +} + +TEST(TestRunnerStandaloneTest, + CustomActivationInterfaceWithInputsReturnsError) { + ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, + DefaultCompiler().Compile("x + y")); + CheckedExpr checked_expr; + ASSERT_THAT(cel::AstToCheckedExpr(*validation_result.GetAst(), &checked_expr), + absl_testing::IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + CreateTestRuntime()); + std::unique_ptr context = + CelTestContext::CreateFromRuntime(std::move(runtime)); + context->SetActivationFactory( + [](const TestCase& test_case, google::protobuf::Arena* arena) + -> absl::StatusOr> { + return std::make_unique(); + }); + context->SetExpressionSource( + CelExpressionSource::FromCheckedExpr(std::move(checked_expr))); + + static auto* test_case_ptr = new TestCase(ParseTextProtoOrDie(R"pb( + input { + key: "x" + value { value { int64_value: 4 } } + } + output { result_value { int64_value: 300 } } + )pb")); + static auto* test_runner_ptr = new TestRunner(std::move(context)); + EXPECT_FATAL_FAILURE( + test_runner_ptr->RunTest(*test_case_ptr), + "Custom bindings or test case input bindings cannot be combined"); +} + TEST(TestRunnerStandaloneTest, CustomAssertFnIsUsed) { // Compile the expression. ASSERT_OK_AND_ASSIGN(cel::ValidationResult validation_result, From 08cc26368d25273ca1b1d31ecd26fdfcb4e5b66d Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 29 Jul 2026 13:09:52 -0700 Subject: [PATCH 154/350] Add range check and configurable limits for lists.range() PiperOrigin-RevId: 956047173 --- extensions/BUILD | 1 + extensions/lists_functions.cc | 73 +++++++++++++++++++------- extensions/lists_functions.h | 47 ++++++++++++++--- extensions/lists_functions_test.cc | 84 ++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 26 deletions(-) diff --git a/extensions/BUILD b/extensions/BUILD index 05104a4a5..a60943f61 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -470,6 +470,7 @@ cc_test( ":lists_functions", "//checker:type_check_issue", "//checker:validation_result", + "//common:ast", "//common:source", "//common:value", "//common:value_testing", diff --git a/extensions/lists_functions.cc b/extensions/lists_functions.cc index bfe05d887..7a40a0387 100644 --- a/extensions/lists_functions.cc +++ b/extensions/lists_functions.cc @@ -68,6 +68,8 @@ absl::Span SortableTypes() { return kTypes; } +constexpr int64_t kMaxRangeSize = 1000000; + // Slow distinct() implementation that uses Equal() to compare values in O(n^2). absl::Status ListDistinctHeterogeneousImpl( const ListValue& list, @@ -223,10 +225,20 @@ absl::StatusOr ListFlatten( return std::move(*builder).Build(); } -absl::StatusOr ListRange( - int64_t end, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, +absl::StatusOr ListRange( + int64_t end, int64_t max_range_size, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { + if (end < 0) { + return ErrorValue(absl::InvalidArgumentError(absl::StrFormat( + "lists.range: size must be non-negative, got %d", end))); + } + if (end > max_range_size) { + return ErrorValue(absl::InvalidArgumentError( + absl::StrFormat("lists.range: size %d exceeds maximum allowed (%d)", + end, max_range_size))); + } auto builder = NewListValueBuilder(arena); builder->Reserve(end); for (int64_t i = 0; i < end; ++i) { @@ -512,11 +524,27 @@ absl::Status RegisterListFlattenFunction(FunctionRegistry& registry) { return absl::OkStatus(); } -absl::Status RegisterListRangeFunction(FunctionRegistry& registry) { - return UnaryFunctionAdapter, - int64_t>::RegisterGlobalOverload("lists.range", - &ListRange, - registry); +absl::Status RegisterListRangeFunction( + FunctionRegistry& registry, + const ListsExtensionOptions& extension_options) { + constexpr int64_t kMaxRangeSize = 1000000; + int64_t effective_limit = kMaxRangeSize; + if (extension_options.max_range_size > 0 && + extension_options.max_range_size < effective_limit) { + effective_limit = extension_options.max_range_size; + } + return UnaryFunctionAdapter, int64_t>:: + RegisterGlobalOverload( + "lists.range", + [effective_limit]( + int64_t end, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) -> absl::StatusOr { + return ListRange(end, effective_limit, descriptor_pool, + message_factory, arena); + }, + registry); } absl::Status RegisterListReverseFunction(FunctionRegistry& registry) { @@ -657,23 +685,23 @@ absl::Status ConfigureParser(ParserBuilder& builder, int version) { } // namespace -absl::Status RegisterListsFunctions(FunctionRegistry& registry, - const RuntimeOptions& options, - int version) { +absl::Status RegisterListsFunctions( + FunctionRegistry& registry, const RuntimeOptions& options, + const ListsExtensionOptions& extension_options) { CEL_RETURN_IF_ERROR(RegisterListSliceFunction(registry)); - if (version == 0) { + if (extension_options.version == 0) { return absl::OkStatus(); } // Since version 1 CEL_RETURN_IF_ERROR(RegisterListFlattenFunction(registry)); - if (version == 1) { + if (extension_options.version == 1) { return absl::OkStatus(); } // Since version 2 CEL_RETURN_IF_ERROR(RegisterListDistinctFunction(registry)); - CEL_RETURN_IF_ERROR(RegisterListRangeFunction(registry)); + CEL_RETURN_IF_ERROR(RegisterListRangeFunction(registry, extension_options)); CEL_RETURN_IF_ERROR(RegisterListReverseFunction(registry)); CEL_RETURN_IF_ERROR(RegisterListSortFunction(registry)); return absl::OkStatus(); @@ -684,18 +712,23 @@ absl::Status RegisterListsMacros(MacroRegistry& registry, const ParserOptions&, return registry.RegisterMacros(lists_macros(version)); } -CheckerLibrary ListsCheckerLibrary(int version) { +CheckerLibrary ListsCheckerLibrary( + const ListsExtensionOptions& extension_options) { return {.id = "cel.lib.ext.lists", - .configure = [version](TypeCheckerBuilder& builder) { + .configure = [version = extension_options.version]( + TypeCheckerBuilder& builder) { return RegisterListsCheckerDecls(builder, version); }}; } -CompilerLibrary ListsCompilerLibrary(int version) { - auto lib = CompilerLibrary::FromCheckerLibrary(ListsCheckerLibrary(version)); - lib.configure_parser = [version](ParserBuilder& builder) { - return ConfigureParser(builder, version); - }; +CompilerLibrary ListsCompilerLibrary( + const ListsExtensionOptions& extension_options) { + auto lib = CompilerLibrary::FromCheckerLibrary( + ListsCheckerLibrary(extension_options)); + lib.configure_parser = + [version = extension_options.version](ParserBuilder& builder) { + return ConfigureParser(builder, version); + }; return lib; } diff --git a/extensions/lists_functions.h b/extensions/lists_functions.h index 0b057170f..a1dd514d2 100644 --- a/extensions/lists_functions.h +++ b/extensions/lists_functions.h @@ -15,6 +15,8 @@ #ifndef THIRD_PARTY_CEL_CPP_EXTENSIONS_LISTS_FUNCTIONS_H_ #define THIRD_PARTY_CEL_CPP_EXTENSIONS_LISTS_FUNCTIONS_H_ +#include + #include "absl/status/status.h" #include "checker/type_checker_builder.h" #include "compiler/compiler.h" @@ -27,6 +29,18 @@ namespace cel::extensions { constexpr int kListsExtensionLatestVersion = 2; +struct ListsExtensionOptions { + int version = kListsExtensionLatestVersion; + + // Maximum size allowed for lists.range(). + // Setting a tighter limit (e.g. 100) will restrict the max size further. + // A standard limit of 1,000,000 applies if a tighter limit isn't + // configured. + int64_t max_range_size = 1000000; +}; + +using ListsFunctionsOptions = ListsExtensionOptions; + // Register implementations for list extension functions. // // === Since version 0 === @@ -45,9 +59,17 @@ constexpr int kListsExtensionLatestVersion = 2; // // .sort() -> list(T) // -absl::Status RegisterListsFunctions(FunctionRegistry& registry, - const RuntimeOptions& options, - int version = kListsExtensionLatestVersion); +absl::Status RegisterListsFunctions( + FunctionRegistry& registry, const RuntimeOptions& options, + const ListsExtensionOptions& extension_options = {}); + +inline absl::Status RegisterListsFunctions(FunctionRegistry& registry, + const RuntimeOptions& options, + int version) { + ListsExtensionOptions extension_options; + extension_options.version = version; + return RegisterListsFunctions(registry, options, extension_options); +} // Register list macros. // @@ -76,7 +98,14 @@ absl::Status RegisterListsMacros(MacroRegistry& registry, // .reverse() -> list(T) // // .sort() -> list(T_) where T_ is partially orderable -CheckerLibrary ListsCheckerLibrary(int version = kListsExtensionLatestVersion); +CheckerLibrary ListsCheckerLibrary( + const ListsExtensionOptions& extension_options = {}); + +inline CheckerLibrary ListsCheckerLibrary(int version) { + ListsExtensionOptions extension_options; + extension_options.version = version; + return ListsCheckerLibrary(extension_options); +} // Provides decls for the following functions: // @@ -96,8 +125,14 @@ CheckerLibrary ListsCheckerLibrary(int version = kListsExtensionLatestVersion); // // .sort() -> list(T_) where T_ is partially orderable CompilerLibrary ListsCompilerLibrary( - int version = kListsExtensionLatestVersion); + const ListsExtensionOptions& extension_options = {}); + +inline CompilerLibrary ListsCompilerLibrary(int version) { + ListsExtensionOptions extension_options; + extension_options.version = version; + return ListsCompilerLibrary(extension_options); +} } // namespace cel::extensions -#endif // THIRD_PARTY_CEL_CPP_EXTENSIONS_SETS_FUNCTIONS_H_ +#endif // THIRD_PARTY_CEL_CPP_EXTENSIONS_LISTS_FUNCTIONS_H_ diff --git a/extensions/lists_functions_test.cc b/extensions/lists_functions_test.cc index 8e9a3c3f5..6e5243889 100644 --- a/extensions/lists_functions_test.cc +++ b/extensions/lists_functions_test.cc @@ -26,6 +26,7 @@ #include "absl/strings/string_view.h" #include "checker/type_check_issue.h" #include "checker/validation_result.h" +#include "common/ast.h" #include "common/source.h" #include "common/value.h" #include "common/value_testing.h" @@ -123,6 +124,10 @@ INSTANTIATE_TEST_SUITE_P( // lists.range() {R"cel(lists.range(4) == [0,1,2,3])cel"}, {R"cel(lists.range(0) == [])cel"}, + {R"cel(lists.range(-1))cel", + "lists.range: size must be non-negative, got -1"}, + {R"cel(lists.range(1000001))cel", + "lists.range: size 1000001 exceeds maximum allowed (1000000)"}, // .reverse() {R"cel([5,1,2,3].reverse() == [3,2,1,5])cel"}, @@ -457,5 +462,84 @@ std::vector CreateListsExtensionVersionParams() { INSTANTIATE_TEST_SUITE_P(ListsExtensionVersionTest, ListsExtensionVersionTest, ValuesIn(CreateListsExtensionVersionParams())); +TEST(ListsFunctionsTest, CustomMaxRangeSizeOption) { + ListsExtensionOptions ext_options; + ext_options.max_range_size = 100; + + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + NewCompilerBuilder(internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(compiler_builder->AddLibrary(ListsCompilerLibrary(ext_options)), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, std::move(*compiler_builder).Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("lists.range(101)", "")); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + const auto runtime_options = RuntimeOptions{}; + ASSERT_OK_AND_ASSIGN( + auto runtime_builder, + CreateStandardRuntimeBuilder(internal::GetTestingDescriptorPool(), + runtime_options)); + ASSERT_THAT(RegisterListsFunctions(runtime_builder.function_registry(), + runtime_options, ext_options), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto runtime, std::move(runtime_builder).Build()); + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); + + google::protobuf::Arena arena; + Activation activation; + ASSERT_OK_AND_ASSIGN(Value eval_result, + program->Evaluate(&arena, activation)); + EXPECT_THAT( + eval_result, + ErrorValueIs(StatusIs( + testing::_, + HasSubstr("lists.range: size 101 exceeds maximum allowed (100)")))); +} + +TEST(ListsFunctionsTest, HardCodedLimitAppliesWhenOptionIsLooser) { + ListsExtensionOptions ext_options; + ext_options.max_range_size = 2000000; + + ASSERT_OK_AND_ASSIGN( + auto compiler_builder, + NewCompilerBuilder(internal::GetTestingDescriptorPool())); + ASSERT_THAT(compiler_builder->AddLibrary(StandardCompilerLibrary()), IsOk()); + ASSERT_THAT(compiler_builder->AddLibrary(ListsCompilerLibrary(ext_options)), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto compiler, std::move(*compiler_builder).Build()); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("lists.range(1000001)", "")); + ASSERT_TRUE(result.IsValid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ast, result.ReleaseAst()); + + const auto runtime_options = RuntimeOptions{}; + ASSERT_OK_AND_ASSIGN( + auto runtime_builder, + CreateStandardRuntimeBuilder(internal::GetTestingDescriptorPool(), + runtime_options)); + ASSERT_THAT(RegisterListsFunctions(runtime_builder.function_registry(), + runtime_options, ext_options), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto runtime, std::move(runtime_builder).Build()); + ASSERT_OK_AND_ASSIGN(auto program, runtime->CreateProgram(std::move(ast))); + + google::protobuf::Arena arena; + Activation activation; + ASSERT_OK_AND_ASSIGN(Value eval_result, + program->Evaluate(&arena, activation)); + EXPECT_THAT( + eval_result, + ErrorValueIs(StatusIs( + testing::_, + HasSubstr( + "lists.range: size 1000001 exceeds maximum allowed (1000000)")))); +} + } // namespace } // namespace cel::extensions From 2fbbd6bcef9e57b17eec02a26b0f2d796bda610c Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 29 Jul 2026 15:47:10 -0700 Subject: [PATCH 155/350] Re-enable conformance test for missing overload at runtime. This was fixed previously, but conformance test wasn't re-enabled. PiperOrigin-RevId: 956129371 --- conformance/BUILD | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/conformance/BUILD b/conformance/BUILD index eb129470f..6f11f345b 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -169,10 +169,6 @@ _TESTS_TO_SKIP = [ "timestamps/duration_converters/get_milliseconds", # Broken test cases which should be supported. - # TODO(issues/112): Unbound functions result in empty eval response. - "basic/functions/unbound", - "basic/functions/unbound_is_runtime_error", - # TODO(issues/97): Parse-only qualified variable lookup "x.y" with binding "x.y" or "y" within container "x" fails "fields/qualified_identifier_resolution/qualified_ident,map_field_select,ident_with_longest_prefix_check,qualified_identifier_resolution_unchecked", "namespace/qualified/self_eval_qualified_lookup", @@ -231,12 +227,6 @@ _TESTS_TO_SKIP_LEGACY = _TESTS_TO_SKIP + [ "block_ext/basic/optional_message", ] -_TESTS_TO_SKIP_CHECKED = [ - # block is a post-check optimization that inserts internal variables. The C++ type checker - # needs support for a proper optimizer for this to work. - # "block_ext", -] - _TESTS_TO_SKIP_LEGACY_DASHBOARD = [ # Future features for CEL 1.0 # TODO(issues/119): Strong typing support for enums, specified but not implemented. @@ -268,7 +258,7 @@ gen_conformance_tests( checked = True, data = _ALL_TESTS, modern = True, - skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_MODERN, ) gen_conformance_tests( @@ -276,7 +266,7 @@ gen_conformance_tests( checked = True, data = _ALL_TESTS, modern = False, - skip_tests = _TESTS_TO_SKIP_LEGACY + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_LEGACY, ) # select optimization is only supported for checked expressions. @@ -286,7 +276,7 @@ gen_conformance_tests( data = _ALL_TESTS, modern = False, select_opt = True, - skip_tests = _TESTS_TO_SKIP_LEGACY + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_LEGACY, ) gen_conformance_tests( @@ -295,7 +285,7 @@ gen_conformance_tests( data = _ALL_TESTS, modern = True, select_opt = True, - skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_MODERN, ) gen_conformance_tests( @@ -304,7 +294,7 @@ gen_conformance_tests( data = _ALL_TESTS, enable_variadic_logical_operators = True, modern = True, - skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_MODERN, ) gen_conformance_tests( @@ -313,7 +303,7 @@ gen_conformance_tests( data = _ALL_TESTS, enable_variadic_logical_operators = True, modern = False, - skip_tests = _TESTS_TO_SKIP_LEGACY + _TESTS_TO_SKIP_CHECKED, + skip_tests = _TESTS_TO_SKIP_LEGACY, ) # Generates a bunch of `cc_test` whose names follow the pattern From 725b3353a06a15718d36dff139427c8c5c3a5619 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 31 Jul 2026 10:35:50 -0700 Subject: [PATCH 156/350] Incrementally check code point limit in NewSource Avoids copying the full source if it is outside of the configured limit. cross ref: https://github.com/google/cel-go/pull/1302 PiperOrigin-RevId: 957218893 --- common/BUILD | 2 +- common/source.cc | 43 +++++++++++++++++++++++--- common/source.h | 37 +++++++++++++++++++--- common/source_test.cc | 51 ++++++++++++++++++++++++++++++- compiler/compiler.h | 4 +-- compiler/compiler_factory_test.cc | 19 ++++++++++++ parser/internal/pratt_parser.cc | 8 +++++ parser/internal/pratt_parser.h | 3 ++ parser/parser.cc | 8 +++++ parser/parser_interface.h | 21 +++++++++++++ parser/parser_test.cc | 20 ++++++++++++ 11 files changed, 203 insertions(+), 13 deletions(-) diff --git a/common/BUILD b/common/BUILD index a1d4ac3bd..937a27f45 100644 --- a/common/BUILD +++ b/common/BUILD @@ -640,8 +640,8 @@ cc_test( deps = [ ":source", "//internal:testing", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings:cord", - "@com_google_absl//absl/types:optional", ], ) diff --git a/common/source.cc b/common/source.cc index 81b1ada9a..e7e09d919 100644 --- a/common/source.cc +++ b/common/source.cc @@ -307,12 +307,18 @@ struct SourceTextTraits { template absl::StatusOr NewSourceImpl(std::string description, const T& text, - const size_t text_size) { + const size_t text_size, + const size_t max_codepoints) { if (ABSL_PREDICT_FALSE( text_size > static_cast(std::numeric_limits::max()))) { return absl::InvalidArgumentError("expression larger than 2GiB limit"); } + if ((text_size >> 2) > max_codepoints) { + // If byte size is 4 times the codepoint limit, then definitely exceeded. + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } using Traits = SourceTextTraits; size_t index = 0; typename Traits::iterator_type it = Traits::Begin(text); @@ -324,6 +330,10 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, std::vector data32; absl::InlinedVector line_offsets; while (index < text_size) { + if (offset >= max_codepoints) { + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } std::tie(code_point, code_units) = cel::internal::Utf8Decode(it); if (ABSL_PREDICT_FALSE(code_point == cel::internal::kUnicodeReplacementCharacter && @@ -375,6 +385,10 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, std::move(description), std::move(line_offsets), Traits::ToVector(text)); latin1: while (index < text_size) { + if (offset >= max_codepoints) { + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } std::tie(code_point, code_units) = internal::Utf8Decode(it); if (ABSL_PREDICT_FALSE(code_point == internal::kUnicodeReplacementCharacter && @@ -420,6 +434,10 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, std::move(description), std::move(line_offsets), std::move(data8)); basic: while (index < text_size) { + if (offset >= max_codepoints) { + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } std::tie(code_point, code_units) = internal::Utf8Decode(it); if (ABSL_PREDICT_FALSE(code_point == internal::kUnicodeReplacementCharacter && @@ -453,6 +471,10 @@ absl::StatusOr NewSourceImpl(std::string description, const T& text, std::move(description), std::move(line_offsets), std::move(data16)); supplemental: while (index < text_size) { + if (offset >= max_codepoints) { + return absl::InvalidArgumentError(absl::StrCat( + "expression is larger than codepoint limit ", max_codepoints)); + } std::tie(code_point, code_units) = internal::Utf8Decode(it); if (ABSL_PREDICT_FALSE(code_point == internal::kUnicodeReplacementCharacter && @@ -631,16 +653,27 @@ absl::Span SourceSubrange::line_offsets() const { return absl::MakeConstSpan(line_offsets_); } +static size_t ClampLimit(int value) { + if (value < 0) { + return std::numeric_limits::max(); + } + return static_cast(value); +} + absl::StatusOr NewSource(absl::string_view content, - std::string description) { + std::string description, + const SourceOptions& options) { return common_internal::NewSourceImpl(std::move(description), content, - content.size()); + content.size(), + ClampLimit(options.max_codepoint_size)); } absl::StatusOr NewSource(const absl::Cord& content, - std::string description) { + std::string description, + const SourceOptions& options) { return common_internal::NewSourceImpl(std::move(description), content, - content.size()); + content.size(), + ClampLimit(options.max_codepoint_size)); } } // namespace cel diff --git a/common/source.h b/common/source.h index b3968453f..190c750cc 100644 --- a/common/source.h +++ b/common/source.h @@ -221,11 +221,40 @@ class SourceSubrange final : public Source { using SourcePtr = std::unique_ptr; -absl::StatusOr NewSource( - absl::string_view content, std::string description = ""); +struct SourceOptions { + // The maximum number of code points allowed in the source. + // A negative value indicates no limit (though still limited by + // int32_t max value). + int max_codepoint_size = 100'000; +}; + +absl::StatusOr NewSource(absl::string_view content, + std::string description, + const SourceOptions& options); + +absl::StatusOr NewSource(const absl::Cord& content, + std::string description, + const SourceOptions& options); + +inline absl::StatusOr NewSource( + absl::string_view content, std::string description) { + return NewSource(content, std::move(description), SourceOptions{}); +} -absl::StatusOr NewSource( - const absl::Cord& content, std::string description = ""); +inline absl::StatusOr NewSource( + const absl::Cord& content, std::string description) { + return NewSource(content, std::move(description), SourceOptions{}); +} + +inline absl::StatusOr NewSource( + absl::string_view content) { + return NewSource(content, "", SourceOptions{}); +} + +inline absl::StatusOr NewSource( + const absl::Cord& content) { + return NewSource(content, "", SourceOptions{}); +} } // namespace cel diff --git a/common/source_test.cc b/common/source_test.cc index e620d3dd8..8315fd517 100644 --- a/common/source_test.cc +++ b/common/source_test.cc @@ -14,8 +14,11 @@ #include "common/source.h" +#include +#include + +#include "absl/status/status.h" #include "absl/strings/cord.h" -#include "absl/types/optional.h" #include "internal/testing.h" namespace cel { @@ -262,5 +265,51 @@ TEST(SourceSubrange, LineOffsetsMiddleSubrange) { EXPECT_THAT(subrange.line_offsets(), ElementsAre(6, 10)); } +TEST(StringSource, CodepointLimitExceeded) { + SourceOptions options; + options.max_codepoint_size = 5; + + EXPECT_THAT( + NewSource("123456", "test", options), + ::absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + ::testing::HasSubstr("expression is larger than codepoint limit 5"))); + + ASSERT_OK_AND_ASSIGN(auto source, NewSource("12345", "test", options)); + EXPECT_THAT(source->content().ToString(), ::testing::Eq("12345")); +} + +TEST(StringSource, CodepointLimitMultibyteUtf8) { + SourceOptions options; + options.max_codepoint_size = 5; + + // "Hello" consists of 5 full-width Unicode characters (15 bytes in + // UTF-8). + ASSERT_OK_AND_ASSIGN(auto source, NewSource("Hello", "test", options)); + EXPECT_THAT(source->content().ToString(), ::testing::Eq("Hello")); + + options.max_codepoint_size = 4; + EXPECT_THAT( + NewSource("Hello", "test", options), + ::absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + ::testing::HasSubstr("expression is larger than codepoint limit 4"))); +} + +TEST(CordSource, CodepointLimitExceeded) { + SourceOptions options; + options.max_codepoint_size = 5; + + EXPECT_THAT( + NewSource(absl::Cord("123456"), "test", options), + ::absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + ::testing::HasSubstr("expression is larger than codepoint limit 5"))); + + ASSERT_OK_AND_ASSIGN(auto source, + NewSource(absl::Cord("12345"), "test", options)); + EXPECT_THAT(source->content().ToString(), ::testing::Eq("12345")); +} + } // namespace } // namespace cel diff --git a/compiler/compiler.h b/compiler/compiler.h index 174912f71..58c60027b 100644 --- a/compiler/compiler.h +++ b/compiler/compiler.h @@ -178,8 +178,8 @@ class Compiler { inline absl::StatusOr Compiler::Compile( absl::string_view source, absl::string_view description, google::protobuf::Arena* absl_nullable arena) const { - absl::StatusOr source_obj = - NewSource(source, std::string(description)); + absl::StatusOr> source_obj = + GetParser().PrepareSource(source, description); if (!source_obj.ok()) { return source_obj.status(); } diff --git a/compiler/compiler_factory_test.cc b/compiler/compiler_factory_test.cc index 3f83f5bf2..f90cc4862 100644 --- a/compiler/compiler_factory_test.cc +++ b/compiler/compiler_factory_test.cc @@ -441,5 +441,24 @@ TEST(CompilerFactoryTest, CompileSourceOverload) { EXPECT_TRUE(result.IsValid()); } +TEST(CompilerFactoryTest, CodepointLimitExceeded) { + CompilerOptions options; + options.parser_options.expression_size_codepoint_limit = 10; + ASSERT_OK_AND_ASSIGN( + auto builder, + NewCompilerBuilder(cel::internal::GetSharedTestingDescriptorPool(), + options)); + ASSERT_OK_AND_ASSIGN(auto compiler, builder->Build()); + + EXPECT_THAT( + compiler->Compile("123456789012345", "test.cel"), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("expression is larger than codepoint limit 10"))); + + ASSERT_OK_AND_ASSIGN(ValidationResult result, + compiler->Compile("1234567890", "test.cel")); + EXPECT_TRUE(result.IsValid()); +} + } // namespace } // namespace cel diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc index 0ccc28c15..3f59e5065 100644 --- a/parser/internal/pratt_parser.cc +++ b/parser/internal/pratt_parser.cc @@ -163,6 +163,14 @@ absl::StatusOr> PrattParserImpl::ParseImpl( return PrattParseImpl(source, macro_registry_, options_, parse_issues); } +absl::StatusOr> PrattParserImpl::PrepareSourceImpl( + absl::string_view input, absl::string_view description) const { + return cel::NewSource( + input, std::string(description), + cel::SourceOptions{.max_codepoint_size = + options_.expression_size_codepoint_limit}); +} + absl::StatusOr> PrattParseImpl( const cel::Source& source, const cel::MacroRegistry& registry, const ParserOptions& options, std::vector* parse_issues) { diff --git a/parser/internal/pratt_parser.h b/parser/internal/pratt_parser.h index 469f48aec..006a10c28 100644 --- a/parser/internal/pratt_parser.h +++ b/parser/internal/pratt_parser.h @@ -59,6 +59,9 @@ class PrattParserImpl final : public cel::Parser { const cel::Source& source, std::vector* absl_nullable parse_issues) const override; + absl::StatusOr> PrepareSourceImpl( + absl::string_view input, absl::string_view description) const override; + std::unique_ptr ToBuilder() const override; private: diff --git a/parser/parser.cc b/parser/parser.cc index 9bc471fdf..28923e02d 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -1781,6 +1781,14 @@ class ParserImpl : public cel::Parser { std::move(parse_result.source_info)); } + absl::StatusOr> PrepareSourceImpl( + absl::string_view input, absl::string_view description) const override { + return cel::NewSource( + input, std::string(description), + cel::SourceOptions{.max_codepoint_size = + options_.expression_size_codepoint_limit}); + } + std::unique_ptr ToBuilder() const override; private: diff --git a/parser/parser_interface.h b/parser/parser_interface.h index ad6e8ca84..eb902c041 100644 --- a/parser/parser_interface.h +++ b/parser/parser_interface.h @@ -114,6 +114,14 @@ class Parser { absl::StatusOr> Parse( const cel::Source& source, std::vector* issues) const; + // Returns a Source object from the given input. + // Validates that the input is well-formed utf-8 and within the configured + // source limits. + absl::StatusOr> PrepareSource( + absl::string_view input, absl::string_view description) const; + absl::StatusOr> PrepareSource( + absl::string_view input) const; + // Returns a builder initialized with the configuration of this parser. virtual std::unique_ptr ToBuilder() const = 0; @@ -121,6 +129,9 @@ class Parser { virtual absl::StatusOr> ParseImpl( const cel::Source& source, std::vector* absl_nullable parse_issues) const = 0; + + virtual absl::StatusOr> PrepareSourceImpl( + absl::string_view input, absl::string_view description) const = 0; }; inline absl::StatusOr> Parser::Parse( @@ -134,6 +145,16 @@ inline absl::StatusOr> Parser::Parse( return ParseImpl(source, issues); } +inline absl::StatusOr> Parser::PrepareSource( + absl::string_view input, absl::string_view description) const { + return PrepareSourceImpl(input, description); +} + +inline absl::StatusOr> Parser::PrepareSource( + absl::string_view input) const { + return PrepareSourceImpl(input, ""); +} + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_PARSER_PARSER_INTERFACE_H_ diff --git a/parser/parser_test.cc b/parser/parser_test.cc index c2fd36112..c04d819b2 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -2189,6 +2189,26 @@ TEST_P(ParserTest, MacroExpansionNodeLimitNotExceeded) { EXPECT_THAT(issues, testing::IsEmpty()); } +TEST_P(ParserTest, PrepareSourceForwardsCodepointLimit) { + auto builder = cel::NewParserBuilder(options_); + builder->GetOptions().expression_size_codepoint_limit = 10; + ASSERT_OK_AND_ASSIGN(auto parser, std::move(*builder).Build()); + + EXPECT_THAT(parser->PrepareSource("123456789012345", "test.cel"), + absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + HasSubstr("expression is larger than codepoint limit 10"))); + + ASSERT_OK_AND_ASSIGN(auto source, + parser->PrepareSource("1234567890", "test.cel")); + EXPECT_EQ(source->description(), "test.cel"); + EXPECT_EQ(source->content().ToString(), "1234567890"); + + ASSERT_OK_AND_ASSIGN(auto source_default_desc, + parser->PrepareSource("1234567890")); + EXPECT_EQ(source_default_desc->description(), ""); +} + std::string ExpressionTestName( const testing::TestParamInfo>& test_info) { const TestInfo& info = std::get<0>(test_info.param); From d0584b6e04b58eab0d3611942591909a0ae6343c Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Fri, 31 Jul 2026 16:09:01 -0700 Subject: [PATCH 157/350] Updated MessageToJson conversion for FieldMask in CEL C++ PiperOrigin-RevId: 957378821 --- common/BUILD | 1 + common/values/parsed_message_value.cc | 19 +---- common/values/parsed_message_value_test.cc | 15 +++- conformance/BUILD | 1 - eval/public/structs/BUILD | 9 +-- eval/public/structs/cel_proto_wrap_util.cc | 59 +++++++++++++- .../structs/cel_proto_wrap_util_test.cc | 79 ++++++++++++++++++- eval/public/structs/cel_proto_wrapper_test.cc | 4 +- 8 files changed, 155 insertions(+), 32 deletions(-) diff --git a/common/BUILD b/common/BUILD index 937a27f45..818854692 100644 --- a/common/BUILD +++ b/common/BUILD @@ -886,6 +886,7 @@ cc_test( "@com_google_absl//absl/time", "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", "@com_google_protobuf//:type_cc_proto", diff --git a/common/values/parsed_message_value.cc b/common/values/parsed_message_value.cc index 8a2b8030d..a2d02dfd0 100644 --- a/common/values/parsed_message_value.cc +++ b/common/values/parsed_message_value.cc @@ -34,12 +34,11 @@ #include "base/attribute.h" #include "common/memory.h" #include "common/value.h" +#include "common/values/values.h" #include "extensions/protobuf/internal/qualify.h" -#include "internal/empty_descriptors.h" #include "internal/json.h" #include "internal/message_equality.h" #include "internal/status_macros.h" -#include "internal/well_known_types.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -51,8 +50,6 @@ namespace cel { namespace { -using ::cel::well_known_types::ValueReflection; - template std::enable_if_t, const google::protobuf::Message* absl_nonnull> @@ -60,14 +57,6 @@ EmptyParsedMessageValue() { return &T::default_instance(); } -template -std::enable_if_t< - std::conjunction_v, - std::negation>>, - const google::protobuf::Message* absl_nonnull> -EmptyParsedMessageValue() { - return internal::GetEmptyDefaultInstance(); -} } // namespace @@ -114,12 +103,8 @@ absl::Status ParsedMessageValue::ConvertToJson( ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); - ValueReflection value_reflection; - CEL_RETURN_IF_ERROR(value_reflection.Initialize(json->GetDescriptor())); - google::protobuf::Message* json_object = value_reflection.MutableStructValue(json); - return internal::MessageToJson(*value_, descriptor_pool, message_factory, - json_object); + json); } absl::Status ParsedMessageValue::ConvertToJsonObject( diff --git a/common/values/parsed_message_value_test.cc b/common/values/parsed_message_value_test.cc index 7a84f82ba..14c76f684 100644 --- a/common/values/parsed_message_value_test.cc +++ b/common/values/parsed_message_value_test.cc @@ -14,9 +14,9 @@ #include +#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "absl/status/status_matchers.h" -#include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "common/memory.h" #include "common/type.h" @@ -74,6 +74,19 @@ TEST_F(ParsedMessageValueTest, SerializeTo) { EXPECT_THAT(std::move(output).Consume(), IsEmpty()); } +TEST_F(ParsedMessageValueTest, ConvertToJsonFieldMask) { + ParsedMessageValue value = + MakeParsedMessage(R"pb(paths: "foo.bar" + paths: "baz")pb"); + google::protobuf::Message* json = + DynamicParseTextProto(R"pb()pb"); + ASSERT_THAT(value.ConvertToJson(descriptor_pool(), message_factory(), + cel::to_address(json)), + IsOk()); + EXPECT_THAT(*json, EqualsTextProto( + R"pb(string_value: "foo.bar,baz")pb")); +} + TEST_F(ParsedMessageValueTest, ConvertToJson) { MessageValue value = MakeParsedMessage(); auto json = DynamicParseTextProto(R"pb()pb"); diff --git a/conformance/BUILD b/conformance/BUILD index 6f11f345b..6bd2dd6ac 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -177,7 +177,6 @@ _TESTS_TO_SKIP = [ "enums/legacy_proto2/select_big,select_neg", # Skip until fixed. - "wrappers/field_mask/to_json", "wrappers/empty/to_json", "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", "parse/receiver_function_names", diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 4e4d5481c..504f8aa7f 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -71,6 +71,7 @@ cc_library( "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -96,24 +97,22 @@ cc_test( ], deps = [ ":cel_proto_wrap_util", - ":protobuf_value_factory", ":trivial_legacy_type_info", "//eval/public:cel_value", - "//eval/public:message_wrapper", "//eval/public/containers:container_backed_list_impl", "//eval/public/containers:container_backed_map_impl", "//eval/testutil:test_message_cc_proto", "//internal:proto_time_encoding", - "//internal:status_macros", "//internal:testing", "//testutil:util", - "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_absl//absl/types:span", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", + "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", "@com_google_protobuf//:wrappers_cc_proto", @@ -211,13 +210,13 @@ cc_test( "//eval/public/containers:container_backed_map_impl", "//eval/testutil:test_message_cc_proto", "//internal:proto_time_encoding", - "//internal:status_macros", "//internal:testing", "//testutil:util", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_absl//absl/types:span", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", diff --git a/eval/public/structs/cel_proto_wrap_util.cc b/eval/public/structs/cel_proto_wrap_util.cc index 7bfe81fe6..c06e0e32e 100644 --- a/eval/public/structs/cel_proto_wrap_util.cc +++ b/eval/public/structs/cel_proto_wrap_util.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include "absl/base/optimization.h" #include "absl/functional/overload.h" #include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" @@ -50,6 +52,7 @@ #include "internal/well_known_types.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/json/json.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -79,7 +82,8 @@ using google::protobuf::Descriptor; using google::protobuf::DescriptorPool; using google::protobuf::Message; using google::protobuf::MessageFactory; - +using google::protobuf::json::MessageToJsonString; +using google::protobuf::json::PrintOptions; // kMaxIntJSON is defined as the Number.MAX_SAFE_INTEGER value per EcmaScript 6. constexpr int64_t kMaxIntJSON = (1ll << 53) - 1; @@ -98,6 +102,27 @@ static bool IsJSONSafe(uint64_t i) { return i <= static_cast(kMaxIntJSON); } +static std::optional GetFieldMaskJsonString( + const google::protobuf::Message& message) { + // TODO(b/540507668): Refactor to pipe descriptor_pool through + // ValueFromValue to use internal::MessageToJson. + PrintOptions json_options; + std::string json_str; + auto status = MessageToJsonString(message, &json_str, json_options); + if (!status.ok()) { + ABSL_LOG(ERROR) << "Failed to convert FieldMask to JSON: " << status; + return std::nullopt; + } + // If JSON marshalling is correct, we know we'll always get a plain + // JSON string value and it shouldn't contain any escapes that we need + // to interpret. + if (json_str.size() >= 2 && json_str.front() == '"' && + json_str.back() == '"') { + return json_str.substr(1, json_str.size() - 2); + } + return json_str; +} + // Map implementation wrapping google.protobuf.ListValue class DynamicList : public CelList { public: @@ -1079,6 +1104,20 @@ google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, co return message; } } break; + case CelValue::Type::kMessage: { + const google::protobuf::Message* message_ptr = value.MessageOrDie(); + if (message_ptr->GetDescriptor()->full_name() == + "google.protobuf.FieldMask") { + std::optional fm_str = + GetFieldMaskJsonString(*message_ptr); + if (fm_str.has_value()) { + reflection.SetStringValue(message, *fm_str); + return message; + } + return nullptr; + } + return nullptr; + } break; case CelValue::Type::kNullType: reflection.SetNullValue(message); return message; @@ -1229,6 +1268,20 @@ bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* return ListFromValue(json->mutable_list_value(), value, arena); case CelValue::Type::kMap: return StructFromValue(json->mutable_struct_value(), value, arena); + case CelValue::Type::kMessage: { + const google::protobuf::Message* message_ptr = value.MessageOrDie(); + if (message_ptr->GetDescriptor()->full_name() == + "google.protobuf.FieldMask") { + std::optional fm_str = + GetFieldMaskJsonString(*message_ptr); + if (fm_str.has_value()) { + json->set_string_value(*fm_str); + return true; + } + return false; + } + return false; + } case CelValue::Type::kNullType: json->set_null_value(protobuf::NULL_VALUE); return true; @@ -1254,7 +1307,7 @@ google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototy case CelValue::Type::kBytes: { BytesValue v; type_name = v.GetTypeName(); - v.set_value(std::string(value.BytesOrDie().value())); + v.set_value(value.BytesOrDie().value()); payload = v.SerializeAsCord(); } break; case CelValue::Type::kDouble: { @@ -1280,7 +1333,7 @@ google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototy case CelValue::Type::kString: { StringValue v; type_name = v.GetTypeName(); - v.set_value(std::string(value.StringOrDie().value())); + v.set_value(value.StringOrDie().value()); payload = v.SerializeAsCord(); } break; case CelValue::Type::kTimestamp: { diff --git a/eval/public/structs/cel_proto_wrap_util_test.cc b/eval/public/structs/cel_proto_wrap_util_test.cc index 59597fe8f..703801198 100644 --- a/eval/public/structs/cel_proto_wrap_util_test.cc +++ b/eval/public/structs/cel_proto_wrap_util_test.cc @@ -15,6 +15,7 @@ #include "eval/public/structs/cel_proto_wrap_util.h" #include +#include #include #include #include @@ -24,30 +25,32 @@ #include "google/protobuf/any.pb.h" #include "google/protobuf/duration.pb.h" #include "google/protobuf/empty.pb.h" +#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "google/protobuf/wrappers.pb.h" -#include "absl/base/no_destructor.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" +#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" -#include "eval/public/message_wrapper.h" -#include "eval/public/structs/protobuf_value_factory.h" #include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/testutil/test_message.pb.h" #include "internal/proto_time_encoding.h" -#include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" +#include "google/protobuf/arena.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" namespace google::api::expr::runtime::internal { namespace { +using ::google::protobuf::FieldMask; +using ::google::protobuf::TextFormat; using ::testing::Eq; using ::testing::UnorderedPointwise; @@ -436,6 +439,74 @@ TEST_F(CelProtoWrapperTest, UnwrapInvalidAny) { UnwrapMessageToValue(&any, &ProtobufValueFactoryImpl, arena()).IsError()); } +TEST_F(CelProtoWrapperTest, WrapFieldMaskToValue) { + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = ProtobufValueFactoryImpl(&field_mask); + + Value expected_message; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb(string_value: "foo.bar,baz")pb", + &expected_message)); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapMapWithFieldMaskToAny) { + const std::string kField = "field_mask"; + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = ProtobufValueFactoryImpl(&field_mask); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + CelValue cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + ASSERT_TRUE( + TextFormat::ParseFromString(R"pb( + fields { + key: "field_mask" + value { string_value: "foo.bar,baz" } + } + )pb", + &expected_struct)); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_struct)); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapListWithFieldMaskToAny) { + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = ProtobufValueFactoryImpl(&field_mask); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + ASSERT_TRUE(TextFormat::ParseFromString( + R"pb(values { string_value: "foo.bar,baz" })pb", &expected_list)); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_list)); + + ExpectWrappedMessage(list_value, expected_message); +} + // Test support of google.protobuf.Value wrappers in CelValue. TEST_F(CelProtoWrapperTest, UnwrapBoolWrapper) { bool value = true; diff --git a/eval/public/structs/cel_proto_wrapper_test.cc b/eval/public/structs/cel_proto_wrapper_test.cc index b9fcd6b51..3ec9c9ac7 100644 --- a/eval/public/structs/cel_proto_wrapper_test.cc +++ b/eval/public/structs/cel_proto_wrapper_test.cc @@ -1,6 +1,7 @@ #include "eval/public/structs/cel_proto_wrapper.h" #include +#include #include #include #include @@ -16,14 +17,15 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" +#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" #include "eval/testutil/test_message.pb.h" #include "internal/proto_time_encoding.h" -#include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" +#include "google/protobuf/arena.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" From 1c4f3fc8a97e536622514e799594d7688753155c Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Sat, 1 Aug 2026 11:21:49 -0700 Subject: [PATCH 158/350] rollback: Updated MessageToJson conversion for FieldMask in CEL C++ PiperOrigin-RevId: 957675302 --- common/BUILD | 1 - common/values/parsed_message_value.cc | 19 ++++- common/values/parsed_message_value_test.cc | 15 +--- conformance/BUILD | 1 + eval/public/structs/BUILD | 9 ++- eval/public/structs/cel_proto_wrap_util.cc | 59 +------------- .../structs/cel_proto_wrap_util_test.cc | 79 +------------------ eval/public/structs/cel_proto_wrapper_test.cc | 4 +- 8 files changed, 32 insertions(+), 155 deletions(-) diff --git a/common/BUILD b/common/BUILD index 818854692..937a27f45 100644 --- a/common/BUILD +++ b/common/BUILD @@ -886,7 +886,6 @@ cc_test( "@com_google_absl//absl/time", "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", - "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", "@com_google_protobuf//:type_cc_proto", diff --git a/common/values/parsed_message_value.cc b/common/values/parsed_message_value.cc index a2d02dfd0..8a2b8030d 100644 --- a/common/values/parsed_message_value.cc +++ b/common/values/parsed_message_value.cc @@ -34,11 +34,12 @@ #include "base/attribute.h" #include "common/memory.h" #include "common/value.h" -#include "common/values/values.h" #include "extensions/protobuf/internal/qualify.h" +#include "internal/empty_descriptors.h" #include "internal/json.h" #include "internal/message_equality.h" #include "internal/status_macros.h" +#include "internal/well_known_types.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -50,6 +51,8 @@ namespace cel { namespace { +using ::cel::well_known_types::ValueReflection; + template std::enable_if_t, const google::protobuf::Message* absl_nonnull> @@ -57,6 +60,14 @@ EmptyParsedMessageValue() { return &T::default_instance(); } +template +std::enable_if_t< + std::conjunction_v, + std::negation>>, + const google::protobuf::Message* absl_nonnull> +EmptyParsedMessageValue() { + return internal::GetEmptyDefaultInstance(); +} } // namespace @@ -103,8 +114,12 @@ absl::Status ParsedMessageValue::ConvertToJson( ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); + ValueReflection value_reflection; + CEL_RETURN_IF_ERROR(value_reflection.Initialize(json->GetDescriptor())); + google::protobuf::Message* json_object = value_reflection.MutableStructValue(json); + return internal::MessageToJson(*value_, descriptor_pool, message_factory, - json); + json_object); } absl::Status ParsedMessageValue::ConvertToJsonObject( diff --git a/common/values/parsed_message_value_test.cc b/common/values/parsed_message_value_test.cc index 14c76f684..7a84f82ba 100644 --- a/common/values/parsed_message_value_test.cc +++ b/common/values/parsed_message_value_test.cc @@ -14,9 +14,9 @@ #include -#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "absl/status/status_matchers.h" +#include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "common/memory.h" #include "common/type.h" @@ -74,19 +74,6 @@ TEST_F(ParsedMessageValueTest, SerializeTo) { EXPECT_THAT(std::move(output).Consume(), IsEmpty()); } -TEST_F(ParsedMessageValueTest, ConvertToJsonFieldMask) { - ParsedMessageValue value = - MakeParsedMessage(R"pb(paths: "foo.bar" - paths: "baz")pb"); - google::protobuf::Message* json = - DynamicParseTextProto(R"pb()pb"); - ASSERT_THAT(value.ConvertToJson(descriptor_pool(), message_factory(), - cel::to_address(json)), - IsOk()); - EXPECT_THAT(*json, EqualsTextProto( - R"pb(string_value: "foo.bar,baz")pb")); -} - TEST_F(ParsedMessageValueTest, ConvertToJson) { MessageValue value = MakeParsedMessage(); auto json = DynamicParseTextProto(R"pb()pb"); diff --git a/conformance/BUILD b/conformance/BUILD index 6bd2dd6ac..6f11f345b 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -177,6 +177,7 @@ _TESTS_TO_SKIP = [ "enums/legacy_proto2/select_big,select_neg", # Skip until fixed. + "wrappers/field_mask/to_json", "wrappers/empty/to_json", "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", "parse/receiver_function_names", diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 504f8aa7f..4e4d5481c 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -71,7 +71,6 @@ cc_library( "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -97,22 +96,24 @@ cc_test( ], deps = [ ":cel_proto_wrap_util", + ":protobuf_value_factory", ":trivial_legacy_type_info", "//eval/public:cel_value", + "//eval/public:message_wrapper", "//eval/public/containers:container_backed_list_impl", "//eval/public/containers:container_backed_map_impl", "//eval/testutil:test_message_cc_proto", "//internal:proto_time_encoding", + "//internal:status_macros", "//internal:testing", "//testutil:util", + "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", - "@com_google_absl//absl/types:span", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", - "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", "@com_google_protobuf//:wrappers_cc_proto", @@ -210,13 +211,13 @@ cc_test( "//eval/public/containers:container_backed_map_impl", "//eval/testutil:test_message_cc_proto", "//internal:proto_time_encoding", + "//internal:status_macros", "//internal:testing", "//testutil:util", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", - "@com_google_absl//absl/types:span", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", diff --git a/eval/public/structs/cel_proto_wrap_util.cc b/eval/public/structs/cel_proto_wrap_util.cc index c06e0e32e..7bfe81fe6 100644 --- a/eval/public/structs/cel_proto_wrap_util.cc +++ b/eval/public/structs/cel_proto_wrap_util.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -32,7 +31,6 @@ #include "absl/base/optimization.h" #include "absl/functional/overload.h" #include "absl/log/absl_check.h" -#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" @@ -52,7 +50,6 @@ #include "internal/well_known_types.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" -#include "google/protobuf/json/json.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -82,8 +79,7 @@ using google::protobuf::Descriptor; using google::protobuf::DescriptorPool; using google::protobuf::Message; using google::protobuf::MessageFactory; -using google::protobuf::json::MessageToJsonString; -using google::protobuf::json::PrintOptions; + // kMaxIntJSON is defined as the Number.MAX_SAFE_INTEGER value per EcmaScript 6. constexpr int64_t kMaxIntJSON = (1ll << 53) - 1; @@ -102,27 +98,6 @@ static bool IsJSONSafe(uint64_t i) { return i <= static_cast(kMaxIntJSON); } -static std::optional GetFieldMaskJsonString( - const google::protobuf::Message& message) { - // TODO(b/540507668): Refactor to pipe descriptor_pool through - // ValueFromValue to use internal::MessageToJson. - PrintOptions json_options; - std::string json_str; - auto status = MessageToJsonString(message, &json_str, json_options); - if (!status.ok()) { - ABSL_LOG(ERROR) << "Failed to convert FieldMask to JSON: " << status; - return std::nullopt; - } - // If JSON marshalling is correct, we know we'll always get a plain - // JSON string value and it shouldn't contain any escapes that we need - // to interpret. - if (json_str.size() >= 2 && json_str.front() == '"' && - json_str.back() == '"') { - return json_str.substr(1, json_str.size() - 2); - } - return json_str; -} - // Map implementation wrapping google.protobuf.ListValue class DynamicList : public CelList { public: @@ -1104,20 +1079,6 @@ google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, co return message; } } break; - case CelValue::Type::kMessage: { - const google::protobuf::Message* message_ptr = value.MessageOrDie(); - if (message_ptr->GetDescriptor()->full_name() == - "google.protobuf.FieldMask") { - std::optional fm_str = - GetFieldMaskJsonString(*message_ptr); - if (fm_str.has_value()) { - reflection.SetStringValue(message, *fm_str); - return message; - } - return nullptr; - } - return nullptr; - } break; case CelValue::Type::kNullType: reflection.SetNullValue(message); return message; @@ -1268,20 +1229,6 @@ bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* return ListFromValue(json->mutable_list_value(), value, arena); case CelValue::Type::kMap: return StructFromValue(json->mutable_struct_value(), value, arena); - case CelValue::Type::kMessage: { - const google::protobuf::Message* message_ptr = value.MessageOrDie(); - if (message_ptr->GetDescriptor()->full_name() == - "google.protobuf.FieldMask") { - std::optional fm_str = - GetFieldMaskJsonString(*message_ptr); - if (fm_str.has_value()) { - json->set_string_value(*fm_str); - return true; - } - return false; - } - return false; - } case CelValue::Type::kNullType: json->set_null_value(protobuf::NULL_VALUE); return true; @@ -1307,7 +1254,7 @@ google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototy case CelValue::Type::kBytes: { BytesValue v; type_name = v.GetTypeName(); - v.set_value(value.BytesOrDie().value()); + v.set_value(std::string(value.BytesOrDie().value())); payload = v.SerializeAsCord(); } break; case CelValue::Type::kDouble: { @@ -1333,7 +1280,7 @@ google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototy case CelValue::Type::kString: { StringValue v; type_name = v.GetTypeName(); - v.set_value(value.StringOrDie().value()); + v.set_value(std::string(value.StringOrDie().value())); payload = v.SerializeAsCord(); } break; case CelValue::Type::kTimestamp: { diff --git a/eval/public/structs/cel_proto_wrap_util_test.cc b/eval/public/structs/cel_proto_wrap_util_test.cc index 703801198..59597fe8f 100644 --- a/eval/public/structs/cel_proto_wrap_util_test.cc +++ b/eval/public/structs/cel_proto_wrap_util_test.cc @@ -15,7 +15,6 @@ #include "eval/public/structs/cel_proto_wrap_util.h" #include -#include #include #include #include @@ -25,32 +24,30 @@ #include "google/protobuf/any.pb.h" #include "google/protobuf/duration.pb.h" #include "google/protobuf/empty.pb.h" -#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "google/protobuf/wrappers.pb.h" +#include "absl/base/no_destructor.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" -#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" +#include "eval/public/message_wrapper.h" +#include "eval/public/structs/protobuf_value_factory.h" #include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/testutil/test_message.pb.h" #include "internal/proto_time_encoding.h" +#include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" -#include "google/protobuf/arena.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" -#include "google/protobuf/text_format.h" namespace google::api::expr::runtime::internal { namespace { -using ::google::protobuf::FieldMask; -using ::google::protobuf::TextFormat; using ::testing::Eq; using ::testing::UnorderedPointwise; @@ -439,74 +436,6 @@ TEST_F(CelProtoWrapperTest, UnwrapInvalidAny) { UnwrapMessageToValue(&any, &ProtobufValueFactoryImpl, arena()).IsError()); } -TEST_F(CelProtoWrapperTest, WrapFieldMaskToValue) { - FieldMask field_mask; - ASSERT_TRUE(TextFormat::ParseFromString(R"pb( - paths: "foo.bar" paths: "baz" - )pb", - &field_mask)); - CelValue value = ProtobufValueFactoryImpl(&field_mask); - - Value expected_message; - ASSERT_TRUE(TextFormat::ParseFromString(R"pb(string_value: "foo.bar,baz")pb", - &expected_message)); - - ExpectWrappedMessage(value, expected_message); -} - -TEST_F(CelProtoWrapperTest, WrapMapWithFieldMaskToAny) { - const std::string kField = "field_mask"; - FieldMask field_mask; - ASSERT_TRUE(TextFormat::ParseFromString(R"pb( - paths: "foo.bar" paths: "baz" - )pb", - &field_mask)); - CelValue value = ProtobufValueFactoryImpl(&field_mask); - - std::vector> args = { - {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr cel_map, - CreateContainerBackedMap( - absl::Span>(args.data(), args.size()))); - CelValue cel_value = CelValue::CreateMap(cel_map.get()); - - Struct expected_struct; - ASSERT_TRUE( - TextFormat::ParseFromString(R"pb( - fields { - key: "field_mask" - value { string_value: "foo.bar,baz" } - } - )pb", - &expected_struct)); - Any expected_message; - ASSERT_TRUE(expected_message.PackFrom(expected_struct)); - - ExpectWrappedMessage(cel_value, expected_message); -} - -TEST_F(CelProtoWrapperTest, WrapListWithFieldMaskToAny) { - FieldMask field_mask; - ASSERT_TRUE(TextFormat::ParseFromString(R"pb( - paths: "foo.bar" paths: "baz" - )pb", - &field_mask)); - CelValue value = ProtobufValueFactoryImpl(&field_mask); - - std::vector list_entries = {value}; - ContainerBackedListImpl cel_list(list_entries); - CelValue list_value = CelValue::CreateList(&cel_list); - - ListValue expected_list; - ASSERT_TRUE(TextFormat::ParseFromString( - R"pb(values { string_value: "foo.bar,baz" })pb", &expected_list)); - Any expected_message; - ASSERT_TRUE(expected_message.PackFrom(expected_list)); - - ExpectWrappedMessage(list_value, expected_message); -} - // Test support of google.protobuf.Value wrappers in CelValue. TEST_F(CelProtoWrapperTest, UnwrapBoolWrapper) { bool value = true; diff --git a/eval/public/structs/cel_proto_wrapper_test.cc b/eval/public/structs/cel_proto_wrapper_test.cc index 3ec9c9ac7..b9fcd6b51 100644 --- a/eval/public/structs/cel_proto_wrapper_test.cc +++ b/eval/public/structs/cel_proto_wrapper_test.cc @@ -1,7 +1,6 @@ #include "eval/public/structs/cel_proto_wrapper.h" #include -#include #include #include #include @@ -17,15 +16,14 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" -#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" #include "eval/testutil/test_message.pb.h" #include "internal/proto_time_encoding.h" +#include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" -#include "google/protobuf/arena.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" From 4f1003df813bca58f4f9d9798a1205d27230c25b Mon Sep 17 00:00:00 2001 From: Antoine Pietri Date: Mon, 3 Aug 2026 05:08:13 -0700 Subject: [PATCH 159/350] Pass message_factory to reflection->GetMessage in cel::Value::WrapField. Inside WrapFieldImpl in value.cc, CEL called reflection->GetMessage(*message, field) without passing message_factory as the 3rd argument. When message is a compiled C++ GeneratedMessage, Protobuf reflection defaulted factory to MessageFactory::generated_factory(). If field is an extension whose descriptor lives in a dynamic DescriptorPool, generated_factory()->GetPrototype(extension_desc) returns nullptr, causing a SIGSEGV (@(nil)) in ExtensionSet::GetMessage(). This CL passes message_factory to reflection->GetMessage(*message, field, message_factory) so Protobuf reflection uses CEL's MessageFactory to resolve dynamic extensions. PiperOrigin-RevId: 958311978 --- common/value.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/common/value.cc b/common/value.cc index 6284626da..fdb18d388 100644 --- a/common/value.cc +++ b/common/value.cc @@ -1583,11 +1583,12 @@ Value WrapFieldImpl( } if constexpr (Unsafe::value) { return Value::WrapMessageUnsafe( - &reflection->GetMessage(*message, field), descriptor_pool, - message_factory, arena); + &reflection->GetMessage(*message, field, message_factory), + descriptor_pool, message_factory, arena); } else { - return Value::WrapMessage(&reflection->GetMessage(*message, field), - descriptor_pool, message_factory, arena); + return Value::WrapMessage( + &reflection->GetMessage(*message, field, message_factory), + descriptor_pool, message_factory, arena); } case google::protobuf::FieldDescriptor::TYPE_BYTES: { std::string scratch; From cf7c8193d0d9d727e66c44f427ba01d50a05bf1d Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Mon, 3 Aug 2026 15:42:53 -0700 Subject: [PATCH 160/350] Fix CEL policy YAML parser to use codepoint positions instead of byte positions. PiperOrigin-RevId: 958616629 --- policy/BUILD | 1 + policy/cel_policy_parse_context.cc | 19 ++++++++++-- policy/cel_policy_parse_context.h | 8 +++++ policy/yaml_policy_parser.cc | 49 +++++++++++++++++++++++++++--- policy/yaml_policy_parser_test.cc | 17 +++++++++++ 5 files changed, 88 insertions(+), 6 deletions(-) diff --git a/policy/BUILD b/policy/BUILD index cace76f5b..6a204d07c 100644 --- a/policy/BUILD +++ b/policy/BUILD @@ -84,6 +84,7 @@ cc_library( ":cel_policy_parser", "//common:source", "//internal:status_macros", + "//internal:utf8", "//policy/internal:yaml_string_element_scanner", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/policy/cel_policy_parse_context.cc b/policy/cel_policy_parse_context.cc index 66861d085..a627bfd50 100644 --- a/policy/cel_policy_parse_context.cc +++ b/policy/cel_policy_parse_context.cc @@ -14,12 +14,13 @@ #include "policy/cel_policy_parse_context.h" +#include #include -#include #include #include #include "absl/log/absl_check.h" +#include "common/source.h" #include "policy/cel_policy.h" #include "policy/cel_policy_parse_result.h" @@ -43,7 +44,21 @@ CelPolicyParseResult CelPolicyParseContext::GetResult() { void CelPolicyParseContext::ReportError(CelPolicyElementId element_id, std::string_view message) { - issues_.push_back(CelPolicyIssue(element_id, std::string(message))); + issues_.push_back(CelPolicyIssue(element_id, message)); +} + +SourcePosition CelPolicyParseContext::GetCodepointPosition( + SourcePosition byte_offset) const { + if (byte_offset < 0) { + return -1; + } + if (byte_to_codepoint_mapping_.empty()) { + return byte_offset; + } + if (static_cast(byte_offset) >= byte_to_codepoint_mapping_.size()) { + return byte_to_codepoint_mapping_.back(); + } + return byte_to_codepoint_mapping_[byte_offset]; } } // namespace cel diff --git a/policy/cel_policy_parse_context.h b/policy/cel_policy_parse_context.h index 6482fa1ae..7e9aa3fe3 100644 --- a/policy/cel_policy_parse_context.h +++ b/policy/cel_policy_parse_context.h @@ -20,6 +20,7 @@ #include #include "absl/strings/string_view.h" +#include "common/source.h" #include "policy/cel_policy.h" #include "policy/cel_policy_parse_result.h" @@ -53,11 +54,18 @@ class CelPolicyParseContext { CelPolicyElementId next_element_id() { return next_element_id_++; } + void set_byte_to_codepoint_mapping(std::vector mapping) { + byte_to_codepoint_mapping_ = std::move(mapping); + } + + SourcePosition GetCodepointPosition(SourcePosition byte_offset) const; + private: std::shared_ptr policy_source_; CelPolicyElementId next_element_id_ = 0; std::vector issues_; std::unique_ptr policy_; + std::vector byte_to_codepoint_mapping_; }; } // namespace cel diff --git a/policy/yaml_policy_parser.cc b/policy/yaml_policy_parser.cc index 4b4a37cb9..37dfa1dad 100644 --- a/policy/yaml_policy_parser.cc +++ b/policy/yaml_policy_parser.cc @@ -14,10 +14,12 @@ #include "policy/yaml_policy_parser.h" +#include #include #include #include #include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -25,24 +27,36 @@ #include "absl/strings/string_view.h" #include "common/source.h" #include "internal/status_macros.h" +#include "internal/utf8.h" #include "policy/cel_policy.h" #include "policy/cel_policy_parse_context.h" #include "policy/cel_policy_parse_result.h" #include "policy/cel_policy_parser.h" #include "policy/internal/yaml_string_element_scanner.h" #include "yaml-cpp/exceptions.h" +#include "yaml-cpp/mark.h" #include "yaml-cpp/node/node.h" #include "yaml-cpp/node/parse.h" #include "yaml-cpp/null.h" #include "yaml-cpp/yaml.h" // IWYU pragma: keep namespace cel { +namespace { + +SourcePosition GetMarkCodepointPosition(const CelPolicyParseContext& ctx, + const YAML::Mark& mark) { + if (mark.is_null() || mark.pos < 0) return -1; + return ctx.GetCodepointPosition(mark.pos); +} + +} // namespace CelPolicyElementId YamlPolicyParser::CollectMetadata( CelPolicyParseContext& ctx, const YAML::Node& node) const { CelPolicyElementId element_id = ctx.next_element_id(); if (!node.Mark().is_null()) { - ctx.policy_source().NoteSourcePosition(element_id, node.Mark().pos); + ctx.policy_source().NoteSourcePosition( + element_id, GetMarkCodepointPosition(ctx, node.Mark())); } return element_id; } @@ -62,9 +76,10 @@ std::optional YamlPolicyParser::GetValueString( } if (!node.Mark().is_null() && ctx.policy_source().content() != nullptr) { + SourcePosition codepoint_pos = GetMarkCodepointPosition(ctx, node.Mark()); policy_internal::YamlStringElement element = policy_internal::ScanYamlStringElement( - ctx.policy_source().content()->content(), node.Mark().pos, + ctx.policy_source().content()->content(), codepoint_pos, node.as()); ctx.policy_source().NoteSourcePosition(id, element.starting_position); @@ -87,14 +102,40 @@ absl::Status YamlPolicyParser::ParsePolicy(CelPolicyParseContext& ctx) const { return absl::OkStatus(); } - ctx.policy().set_description(ValueString(-1, source->description())); + // TODO(b/542282964): Fold this mapping into cel::Source decoding happens + // once. std::string text = source->content().ToString(); + std::vector mapping; + mapping.resize(text.size() + 1, 0); + size_t byte_offset = 0; + SourcePosition codepoint = 0; + absl::string_view view = text; + while (!view.empty()) { + auto [code_point, code_units] = cel::internal::Utf8Decode(view); + if (code_units == 0) break; + for (size_t i = 0; i < code_units; ++i) { + if (byte_offset + i < mapping.size()) { + mapping[byte_offset + i] = codepoint; + } + } + byte_offset += code_units; + view.remove_prefix(code_units); + codepoint++; + } + for (size_t i = byte_offset; i < mapping.size(); ++i) { + mapping[i] = codepoint; + } + + ctx.set_byte_to_codepoint_mapping(std::move(mapping)); + + ctx.policy().set_description(ValueString(-1, source->description())); YAML::Node node; try { node = YAML::Load(text); } catch (YAML::Exception& e) { if (!e.mark.is_null()) { - ctx.policy_source().NoteSourcePosition(0, e.mark.pos); + ctx.policy_source().NoteSourcePosition( + 0, GetMarkCodepointPosition(ctx, e.mark)); } ctx.ReportError(0, "Invalid CEL policy YAML syntax"); return absl::OkStatus(); diff --git a/policy/yaml_policy_parser_test.cc b/policy/yaml_policy_parser_test.cc index 4e7dfc49c..f32b7f53a 100644 --- a/policy/yaml_policy_parser_test.cc +++ b/policy/yaml_policy_parser_test.cc @@ -150,6 +150,12 @@ TEST_P(YamlPolicyParseErrorTest, YamlSyntaxError) { std::vector GetParseTestCases() { return { + ParseTestCase{ + .yaml = "name: \"unclosed", + .expected_error = "1:16: Invalid CEL policy YAML syntax\n" + " | name: \"unclosed\n" + " | ...............^", + }, ParseTestCase{ .yaml = R"yaml( ? [ John, Doe ]: age: 30 )yaml", .expected_error = "1:22: Invalid CEL policy YAML syntax\n" @@ -198,6 +204,17 @@ std::vector GetParseTestCases() { " | - cel.expr.conformance\n" " | ....................^", }, + ParseTestCase{ + .yaml = R"yaml( + # Comment with multi-byte char: € + imports: + - name: + - cel.expr.conformance + )yaml", + .expected_error = "5:21: Import name is not a string\n" + " | - cel.expr.conformance\n" + " | ....................^", + }, ParseTestCase{ .yaml = R"yaml( rule: do something From 3aa5ae8067a37bcb0f2b08a5c002c299d95dc7c8 Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Mon, 3 Aug 2026 16:03:17 -0700 Subject: [PATCH 161/350] Internal Cleanup PiperOrigin-RevId: 958626694 --- conformance/BUILD | 3 --- 1 file changed, 3 deletions(-) diff --git a/conformance/BUILD b/conformance/BUILD index 6f11f345b..de1010628 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -173,14 +173,11 @@ _TESTS_TO_SKIP = [ "fields/qualified_identifier_resolution/qualified_ident,map_field_select,ident_with_longest_prefix_check,qualified_identifier_resolution_unchecked", "namespace/qualified/self_eval_qualified_lookup", "namespace/namespace/self_eval_container_lookup,self_eval_container_lookup_unchecked", - # TODO(issues/117): Integer overflow on enum assignments should error. - "enums/legacy_proto2/select_big,select_neg", # Skip until fixed. "wrappers/field_mask/to_json", "wrappers/empty/to_json", "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", - "parse/receiver_function_names", # Future features for CEL 1.0 # TODO(issues/119): Strong typing support for enums, specified but not implemented. From b4fbcaf1ddf85fc6b79b2824aa9e2aba3861a864 Mon Sep 17 00:00:00 2001 From: Antoine Pietri Date: Tue, 4 Aug 2026 06:06:57 -0700 Subject: [PATCH 162/350] Add unit test for cel::Value::WrapField with dynamic message extension fields. Adds TEST(Value, WrapDynamicExtensionMessageField) to value_test.cc to verify that wrapping a dynamic extension message field on a compiled C++ GeneratedMessage using cel::Value::WrapField succeeds when message_factory is passed to GetMessage. PiperOrigin-RevId: 958977840 --- common/BUILD | 1 + common/value_test.cc | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/common/BUILD b/common/BUILD index 937a27f45..eb08dfcc3 100644 --- a/common/BUILD +++ b/common/BUILD @@ -885,6 +885,7 @@ cc_test( "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/time", "@com_google_absl//absl/types:optional", + "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", diff --git a/common/value_test.cc b/common/value_test.cc index 40ca590b0..56cd5252b 100644 --- a/common/value_test.cc +++ b/common/value_test.cc @@ -28,9 +28,11 @@ #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "internal/testing_message_factory.h" +#include "cel/expr/conformance/proto2/test_all_types.pb.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/dynamic_message.h" #include "google/protobuf/generated_enum_reflection.h" namespace cel { @@ -973,6 +975,43 @@ TEST(Value, NumericHeterogeneousEquality) { EXPECT_NE(DoubleValue(1), UintValue(2)); } +TEST(Value, WrapDynamicExtensionMessageField) { + google::protobuf::DescriptorPool pool(google::protobuf::DescriptorPool::generated_pool()); + google::protobuf::FileDescriptorProto file_proto; + file_proto.set_name("dynamic_extension.proto"); + file_proto.set_package("cel.expr.conformance.proto2"); + file_proto.add_dependency( + cel::expr::conformance::proto2::TestAllTypes::descriptor() + ->file() + ->name()); + + auto* msg_type = file_proto.add_message_type(); + msg_type->set_name("DynamicExtensionMessage"); + + auto* ext = file_proto.add_extension(); + ext->set_name("dynamic_ext"); + ext->set_number(1000); + ext->set_label(google::protobuf::FieldDescriptorProto::LABEL_OPTIONAL); + ext->set_type(google::protobuf::FieldDescriptorProto::TYPE_MESSAGE); + ext->set_type_name(".cel.expr.conformance.proto2.DynamicExtensionMessage"); + ext->set_extendee( + cel::expr::conformance::proto2::TestAllTypes::descriptor()->full_name()); + + const google::protobuf::FileDescriptor* file_desc = pool.BuildFile(file_proto); + ASSERT_NE(file_desc, nullptr); + + const google::protobuf::FieldDescriptor* ext_desc = + pool.FindExtensionByName("cel.expr.conformance.proto2.dynamic_ext"); + ASSERT_NE(ext_desc, nullptr); + + google::protobuf::DynamicMessageFactory dynamic_factory(&pool); + google::protobuf::Arena arena; + cel::expr::conformance::proto2::TestAllTypes test_msg; + Value val = + Value::WrapField(&test_msg, ext_desc, &pool, &dynamic_factory, &arena); + EXPECT_TRUE(val.IsMessage()); +} + using ValueIteratorTest = common_internal::ValueTest<>; TEST_F(ValueIteratorTest, Empty) { From 5a38e1cacfce5081d5f426a4c91730601dd7a052 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 4 Aug 2026 08:37:31 -0700 Subject: [PATCH 163/350] rollback: Opt-in to eager field lookup by default. PiperOrigin-RevId: 959041307 --- eval/public/cel_options.h | 2 +- runtime/runtime_options.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index e72b33eb0..001990431 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -222,7 +222,7 @@ struct InterpreterOptions { // When enabled, the planner will attempt to use a more performant execution // path for field access when the type is known at plan time, instead of using // the generic field access implementation. - bool enable_typed_field_access = true; + bool enable_typed_field_access = false; }; // LINT.ThenChange(//depot/google3/runtime/runtime_options.h) diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 1624dfb1c..072daf26c 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -200,7 +200,7 @@ struct RuntimeOptions { // This is not recommended if the values bound to the activation are typically // not what the planner expected (e.g. a map that was declared as a proto or // a different message with matching field names). - bool enable_typed_field_access = true; + bool enable_typed_field_access = false; }; // LINT.ThenChange(//depot/google3/eval/public/cel_options.h) From 72e36169657fdf34b1a98c4ba93289d319f5e078 Mon Sep 17 00:00:00 2001 From: Antoine Pietri Date: Wed, 5 Aug 2026 10:10:09 -0700 Subject: [PATCH 164/350] Deprecate CEL status macros and StatusBuilder in favor of the upstream absl version. - Update `CEL_RETURN_IF_ERROR` and `CEL_ASSIGN_OR_RETURN` to use upstream `absl` status macros. - Mark `CEL_` macros as deprecated with comments. - Deprecate `cel::internal::StatusBuilder` in favor of `absl::StatusBuilder` using `[[deprecated]]` and `ABSL_REFACTOR_INLINE`. - Remove custom OSS implementations from `status_macros.h` and `status_builder.h` as they are now upstream in ABSL. PiperOrigin-RevId: 959727072 --- internal/BUILD | 5 +- internal/status_builder.h | 95 ++------------------------ internal/status_macros.h | 136 ++------------------------------------ internal/testing.h | 5 +- 4 files changed, 17 insertions(+), 224 deletions(-) diff --git a/internal/BUILD b/internal/BUILD index 6d0efab72..189853323 100644 --- a/internal/BUILD +++ b/internal/BUILD @@ -104,6 +104,7 @@ cc_library( deps = [ "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_builder", ], ) @@ -157,9 +158,9 @@ cc_library( name = "status_macros", hdrs = ["status_macros.h"], deps = [ - ":status_builder", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_macros", ], ) @@ -306,6 +307,7 @@ cc_library( "testing.h", ], deps = [ + ":status_builder", ":status_macros", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", @@ -323,6 +325,7 @@ cc_library( "testing.h", ], deps = [ + ":status_builder", ":status_macros", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", diff --git a/internal/status_builder.h b/internal/status_builder.h index 9caa6c462..80cdd7712 100644 --- a/internal/status_builder.h +++ b/internal/status_builder.h @@ -15,99 +15,14 @@ #ifndef THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_BUILDER_H_ #define THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_BUILDER_H_ -#include -#include - -#include "absl/base/attributes.h" -#include "absl/status/status.h" +#include "absl/base/macros.h" +#include "absl/status/status_builder.h" namespace cel::internal { -class StatusBuilder; - -template -inline constexpr bool StatusBuilderResultMatches = - std::is_same_v>, Expected>; - -template -using StatusBuilderPurePolicy = std::enable_if_t< - StatusBuilderResultMatches, - std::invoke_result_t>; - -template -using StatusBuilderSideEffect = - std::enable_if_t, - std::invoke_result_t>; - -template -using StatusBuilderConversion = std::enable_if_t< - !StatusBuilderResultMatches && - !StatusBuilderResultMatches, - std::invoke_result_t>; - -class StatusBuilder final { - public: - StatusBuilder() = default; - - explicit StatusBuilder(const absl::Status& status) : status_(status) {} - - StatusBuilder(const StatusBuilder&) = default; - - StatusBuilder(StatusBuilder&&) = default; - - ~StatusBuilder() = default; - - StatusBuilder& operator=(const StatusBuilder&) = default; - - StatusBuilder& operator=(StatusBuilder&&) = default; - - bool ok() const { return status_.ok(); } - - absl::StatusCode code() const { return status_.code(); } - - operator absl::Status() const& { return status_; } // NOLINT - - operator absl::Status() && { return std::move(status_); } // NOLINT - - template - auto With( - Adaptor&& adaptor) & -> StatusBuilderPurePolicy { - return std::forward(adaptor)(*this); - } - template - ABSL_MUST_USE_RESULT auto With( - Adaptor&& - adaptor) && -> StatusBuilderPurePolicy { - return std::forward(adaptor)(std::move(*this)); - } - - template - auto With( - Adaptor&& adaptor) & -> StatusBuilderSideEffect { - return std::forward(adaptor)(*this); - } - template - ABSL_MUST_USE_RESULT auto With( - Adaptor&& - adaptor) && -> StatusBuilderSideEffect { - return std::forward(adaptor)(std::move(*this)); - } - - template - auto With( - Adaptor&& adaptor) & -> StatusBuilderConversion { - return std::forward(adaptor)(*this); - } - template - ABSL_MUST_USE_RESULT auto With( - Adaptor&& - adaptor) && -> StatusBuilderConversion { - return std::forward(adaptor)(std::move(*this)); - } - - private: - absl::Status status_; -}; +using StatusBuilder + [[deprecated("Use absl::StatusBuilder instead")]] ABSL_REFACTOR_INLINE = + ::absl::StatusBuilder; } // namespace cel::internal diff --git a/internal/status_macros.h b/internal/status_macros.h index a4b662df6..8ab9a4914 100644 --- a/internal/status_macros.h +++ b/internal/status_macros.h @@ -15,138 +15,12 @@ #ifndef THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_MACROS_H_ #define THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_MACROS_H_ -#include +#include "absl/status/status_macros.h" -#include "absl/base/optimization.h" -#include "absl/status/status.h" -#include "internal/status_builder.h" +// Deprecated: use ABSL_RETURN_IF_ERROR directly. +#define CEL_RETURN_IF_ERROR(expr) ABSL_RETURN_IF_ERROR(expr) -#define CEL_RETURN_IF_ERROR(expr) \ - CEL_INTERNAL_STATUS_MACROS_IMPL_ELSE_BLOCKER_ \ - if (::cel::internal::StatusAdaptor cel_internal_status_macro = {(expr)}) { \ - } else /* NOLINT */ \ - return cel_internal_status_macro.Consume() - -// The GNU compiler historically emitted warnings for obscure usages of -// `if (foo) if (bar) {} else`. This suppresses that. - -// clang-format off -#define CEL_INTERNAL_STATUS_MACROS_IMPL_ELSE_BLOCKER_ \ - switch (0) case 0: default: /* NOLINT */ -// clang-format on - -#define CEL_ASSIGN_OR_RETURN(...) \ - CEL_INTERNAL_STATUS_MACROS_GET_VARIADIC_( \ - (__VA_ARGS__, CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_3_, \ - CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_2_)) \ - (__VA_ARGS__) - -// The following are macro magic to select either the 2 arg variant or 3 arg -// variant of CEL_ASSIGN_OR_RETURN. - -#define CEL_INTERNAL_STATUS_MACROS_GET_VARIADIC_HELPER_(_1, _2, _3, NAME, ...) \ - NAME -#define CEL_INTERNAL_STATUS_MACROS_GET_VARIADIC_(args) \ - CEL_INTERNAL_STATUS_MACROS_GET_VARIADIC_HELPER_ args - -#define CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_2_(lhs, rexpr) \ - CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_( \ - CEL_INTERNAL_STATUS_MACROS_CONCAT(_status_or_value, __LINE__), lhs, \ - rexpr, \ - return absl::Status(std::move(CEL_INTERNAL_STATUS_MACROS_CONCAT( \ - _status_or_value, __LINE__)) \ - .status())) - -#define CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_3_(lhs, rexpr, \ - error_expression) \ - CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_( \ - CEL_INTERNAL_STATUS_MACROS_CONCAT(_status_or_value, __LINE__), lhs, \ - rexpr, \ - ::cel::internal::StatusBuilder _( \ - std::move( \ - CEL_INTERNAL_STATUS_MACROS_CONCAT(_status_or_value, __LINE__)) \ - .status()); \ - (void)_; /* error_expression is allowed to not use this variable */ \ - return (error_expression)) - -// Common implementation of CEL_ASSIGN_OR_RETURN. Both the 2 arg variant and 3 -// arg variant are implemented by this macro. - -#define CEL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_(statusor, lhs, rexpr, \ - error_expression) \ - auto statusor = (rexpr); \ - if (ABSL_PREDICT_FALSE(!statusor.ok())) { \ - error_expression; \ - } \ - CEL_INTERNAL_STATUS_MACROS_UNPARENTHESIZE_IF_PARENTHESIZED(lhs) = \ - std::move(statusor).value() - -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER(...) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER_HELPER((__VA_ARGS__, 0, 1)) - -// MSVC historically expands variadic macros incorrectly, so another level of -// indirection is required. -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER_HELPER(args) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER_I args -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER_I(e0, e1, is_empty, ...) \ - is_empty - -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY(...) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_I(__VA_ARGS__) -#define CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_I(...) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY_INNER(_, ##__VA_ARGS__) - -#define CEL_INTERNAL_STATUS_MACROS_IF_1(_Then, _Else) _Then -#define CEL_INTERNAL_STATUS_MACROS_IF_0(_Then, _Else) _Else -#define CEL_INTERNAL_STATUS_MACROS_IF(_Cond, _Then, _Else) \ - CEL_INTERNAL_STATUS_MACROS_CONCAT(CEL_INTERNAL_STATUS_MACROS_IF_, _Cond) \ - (_Then, _Else) - -#define CEL_INTERNAL_STATUS_MACROS_EAT(...) -#define CEL_INTERNAL_STATUS_MACROS_REM(...) __VA_ARGS__ -#define CEL_INTERNAL_STATUS_MACROS_EMPTY() - -// Expands to 1 if the input is surrounded by parenthesis, 0 otherwise. -#define CEL_INTERNAL_STATUS_MACROS_IS_PARENTHESIZED(...) \ - CEL_INTERNAL_STATUS_MACROS_IS_EMPTY( \ - CEL_INTERNAL_STATUS_MACROS_EAT __VA_ARGS__) - -// If the input is surrounded by parenthesis, remove them. Otherwise expand it -// unchanged. -#define CEL_INTERNAL_STATUS_MACROS_UNPARENTHESIZE_IF_PARENTHESIZED(...) \ - CEL_INTERNAL_STATUS_MACROS_IF( \ - CEL_INTERNAL_STATUS_MACROS_IS_PARENTHESIZED(__VA_ARGS__), \ - CEL_INTERNAL_STATUS_MACROS_REM, CEL_INTERNAL_STATUS_MACROS_EMPTY()) \ - __VA_ARGS__ - -#define CEL_INTERNAL_STATUS_MACROS_CONCAT_HELPER(x, y) x##y -#define CEL_INTERNAL_STATUS_MACROS_CONCAT(x, y) \ - CEL_INTERNAL_STATUS_MACROS_CONCAT_HELPER(x, y) - -namespace cel::internal { - -class StatusAdaptor final { - public: - StatusAdaptor() = default; - - StatusAdaptor(const StatusAdaptor&) = delete; - - StatusAdaptor(StatusAdaptor&&) = delete; - - StatusAdaptor(const absl::Status& status) : builder_(status) {} // NOLINT - - StatusAdaptor& operator=(const StatusAdaptor&) = delete; - - StatusAdaptor& operator=(StatusAdaptor&&) = delete; - - StatusBuilder&& Consume() { return std::move(builder_); } - - explicit operator bool() const { return ABSL_PREDICT_TRUE(builder_.ok()); } - - private: - StatusBuilder builder_; -}; - -} // namespace cel::internal +// Deprecated: use ABSL_ASSIGN_OR_RETURN directly. +#define CEL_ASSIGN_OR_RETURN(...) ABSL_ASSIGN_OR_RETURN(__VA_ARGS__) #endif // THIRD_PARTY_CEL_CPP_INTERNAL_STATUS_MACROS_H_ diff --git a/internal/testing.h b/internal/testing.h index e1b9f7498..a909a8c6d 100644 --- a/internal/testing.h +++ b/internal/testing.h @@ -18,14 +18,15 @@ #include "gmock/gmock.h" // IWYU pragma: export #include "gtest/gtest.h" // IWYU pragma: export #include "absl/status/status_matchers.h" +#include "internal/status_builder.h" #include "internal/status_macros.h" // IWYU pragma: keep #ifndef ASSERT_OK -#define ASSERT_OK(expr) ASSERT_THAT(expr, ::absl_testing::IsOk()) +#define ASSERT_OK(expr) ABSL_ASSERT_OK(expr) #endif #ifndef EXPECT_OK -#define EXPECT_OK(expr) EXPECT_THAT(expr, ::absl_testing::IsOk()) +#define EXPECT_OK(expr) ABSL_EXPECT_OK(expr) #endif #ifndef ASSERT_OK_AND_ASSIGN From a2d524040cd890a662e72f7d52bccbcaf0c6b643 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 5 Aug 2026 11:26:44 -0700 Subject: [PATCH 165/350] Prevent new subclasses of legacy TypeInfoApis. This was a work around for custom reflection operations on protos while the cel::Value work was in progress. Stopping new usages now that custom structs can be introduced via cel::CustomStructValue. PiperOrigin-RevId: 959771840 --- eval/eval/BUILD | 4 - eval/eval/select_step_test.cc | 94 -------- eval/public/structs/BUILD | 15 +- eval/public/structs/legacy_type_adapter.h | 19 ++ .../structs/legacy_type_adapter_test.cc | 42 +--- eval/public/structs/legacy_type_info_apis.h | 20 +- eval/public/structs/legacy_type_provider.h | 13 +- .../structs/legacy_type_provider_test.cc | 50 ++--- .../structs/proto_message_type_adapter.cc | 57 +---- extensions/BUILD | 6 +- extensions/select_optimization_test.cc | 202 +++++++++--------- 11 files changed, 174 insertions(+), 348 deletions(-) diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 2597563a4..507ff6f0e 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -789,9 +789,7 @@ cc_test( deps = [ ":attribute_trail", ":cel_expression_flat_impl", - ":compiler_constant_step", ":const_value_step", - ":create_map_step", ":evaluator_core", ":ident_step", ":select_step", @@ -810,9 +808,7 @@ cc_test( "//eval/public:unknown_set", "//eval/public/containers:container_backed_map_impl", "//eval/public/structs:cel_proto_wrapper", - "//eval/public/structs:legacy_type_adapter", "//eval/public/structs:trivial_legacy_type_info", - "//eval/public/testing:matchers", "//eval/testutil:test_extensions_cc_proto", "//eval/testutil:test_message_cc_proto", "//extensions/protobuf:value", diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index 1580472ba..494def40f 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -24,9 +24,7 @@ #include "common/value_testing.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/compiler_constant_step.h" #include "eval/eval/const_value_step.h" -#include "eval/eval/create_map_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/ident_step.h" #include "eval/public/activation.h" @@ -34,9 +32,7 @@ #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_map_impl.h" #include "eval/public/structs/cel_proto_wrapper.h" -#include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/trivial_legacy_type_info.h" -#include "eval/public/testing/matchers.h" #include "eval/public/unknown_attribute_set.h" #include "eval/public/unknown_set.h" #include "eval/testutil/test_extensions.pb.h" @@ -81,10 +77,8 @@ using ::cel::internal::test::EqualsProto; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::cel::test::IntValueIs; -using ::testing::_; using ::testing::Eq; using ::testing::HasSubstr; -using ::testing::Return; using ::testing::UnorderedElementsAre; struct RunExpressionOptions { @@ -92,32 +86,6 @@ struct RunExpressionOptions { bool enable_wrapper_type_null_unboxing = false; }; -// Simple implementation LegacyTypeAccessApis / LegacyTypeInfoApis that allows -// mocking for getters/setters. -class MockAccessor : public LegacyTypeAccessApis, public LegacyTypeInfoApis { - public: - MOCK_METHOD(absl::StatusOr, HasField, - (absl::string_view field_name, - const CelValue::MessageWrapper& value), - (const, override)); - MOCK_METHOD(absl::StatusOr, GetField, - (absl::string_view field_name, - const CelValue::MessageWrapper& instance, - ProtoWrapperTypeOptions unboxing_option, - cel::MemoryManagerRef memory_manager), - (const, override)); - MOCK_METHOD(absl::string_view, GetTypename, - (const CelValue::MessageWrapper& instance), (const, override)); - MOCK_METHOD(std::string, DebugString, - (const CelValue::MessageWrapper& instance), (const, override)); - MOCK_METHOD(std::vector, ListFields, - (const CelValue::MessageWrapper& value), (const, override)); - const LegacyTypeAccessApis* GetAccessApis( - const CelValue::MessageWrapper& instance) const override { - return this; - } -}; - class SelectStepTest : public testing::Test { public: SelectStepTest() : env_(NewTestingRuntimeEnv()) {} @@ -702,68 +670,6 @@ TEST_P(SelectStepConformanceTest, NullMessageAccessor) { EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kNotFound)); } -TEST_P(SelectStepConformanceTest, CustomAccessor) { - TestMessage message; - TestMessage* message2 = message.mutable_message_value(); - message2->set_int32_value(1); - message2->set_string_value("test"); - RunExpressionOptions options; - options.enable_unknowns = GetParam(); - testing::NiceMock accessor; - CelValue value = CelValue::CreateMessageWrapper( - CelValue::MessageWrapper(&message, &accessor)); - - ON_CALL(accessor, GetField(_, _, _, _)) - .WillByDefault(Return(CelValue::CreateInt64(2))); - ON_CALL(accessor, HasField(_, _)).WillByDefault(Return(false)); - - ASSERT_OK_AND_ASSIGN(CelValue result, - RunExpression(value, "message_value", - /*test=*/false, - /*unknown_path=*/"", options)); - - EXPECT_THAT(result, test::IsCelInt64(2)); - - // testonly select (has) - ASSERT_OK_AND_ASSIGN(result, RunExpression(value, "message_value", - /*test=*/true, - /*unknown_path=*/"", options)); - - EXPECT_THAT(result, test::IsCelBool(false)); -} - -TEST_P(SelectStepConformanceTest, CustomAccessorErrorHandling) { - TestMessage message; - TestMessage* message2 = message.mutable_message_value(); - message2->set_int32_value(1); - message2->set_string_value("test"); - RunExpressionOptions options; - options.enable_unknowns = GetParam(); - testing::NiceMock accessor; - CelValue value = CelValue::CreateMessageWrapper( - CelValue::MessageWrapper(&message, &accessor)); - - ON_CALL(accessor, GetField(_, _, _, _)) - .WillByDefault(Return(absl::InternalError("bad data"))); - ON_CALL(accessor, HasField(_, _)) - .WillByDefault(Return(absl::NotFoundError("not found"))); - - // For get field, implementation may return an error-type cel value or a - // status (e.g. broken assumption using a core type). - ASSERT_OK_AND_ASSIGN(CelValue result, - RunExpression(value, "message_value", - /*test=*/false, - /*unknown_path=*/"", options)); - EXPECT_THAT(result, test::IsCelError(StatusIs(absl::StatusCode::kInternal))); - - // testonly select (has) errors are coerced to CelError. - ASSERT_OK_AND_ASSIGN(result, RunExpression(value, "message_value", - /*test=*/true, - /*unknown_path=*/"", options)); - - EXPECT_THAT(result, test::IsCelError(StatusIs(absl::StatusCode::kNotFound))); -} - TEST_P(SelectStepConformanceTest, SimpleEnumTest) { TestMessage message; message.set_enum_value(TestMessage::TEST_ENUM_1); diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 4e4d5481c..75b4685a0 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -272,14 +272,9 @@ cc_test( srcs = ["legacy_type_adapter_test.cc"], deps = [ ":legacy_type_adapter", - ":trivial_legacy_type_info", - "//eval/public:cel_value", - "//eval/public/testing:matchers", + ":proto_message_type_adapter", "//eval/testutil:test_message_cc_proto", - "//extensions/protobuf:memory_manager", - "//internal:status_macros", "//internal:testing", - "@com_google_protobuf//:protobuf", ], ) @@ -301,7 +296,6 @@ cc_library( "//eval/public/containers:internal_field_backed_map_impl", "//extensions/protobuf:memory_manager", "//extensions/protobuf/internal:qualify", - "//internal:casts", "//internal:status_macros", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", @@ -309,7 +303,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_protobuf//:differencer", "@com_google_protobuf//:protobuf", @@ -386,7 +379,6 @@ cc_library( deps = [ "//eval/public:message_wrapper", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status", "@com_google_absl//absl/strings:string_view", "@com_google_protobuf//:protobuf", ], @@ -418,11 +410,14 @@ cc_test( name = "legacy_type_provider_test", srcs = ["legacy_type_provider_test.cc"], deps = [ + ":legacy_type_adapter", ":legacy_type_info_apis", ":legacy_type_provider", + ":proto_message_type_adapter", + ":trivial_legacy_type_info", "//common:type", + "//eval/testutil:test_message_cc_proto", "//internal:testing", - "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings:string_view", ], ) diff --git a/eval/public/structs/legacy_type_adapter.h b/eval/public/structs/legacy_type_adapter.h index dc7a3ab1b..05fbc9a92 100644 --- a/eval/public/structs/legacy_type_adapter.h +++ b/eval/public/structs/legacy_type_adapter.h @@ -32,6 +32,10 @@ namespace google::api::expr::runtime { +// Forward declare permitted subclasses. +class DucktypedMessageAdapter; +class ProtoMessageTypeAdapter; + // Interface for mutation apis. // Note: in the new type system, a type provider represents this by returning // a cel::Type and cel::ValueManager for the type. @@ -71,6 +75,13 @@ class LegacyTypeMutationApis { CelValue::MessageWrapper::Builder& instance [[maybe_unused]]) const { return absl::UnimplementedError("SetFieldByNumber is not yet implemented"); } + + private: + // This class should only be implemented by CEL. Custom structs are only + // supported using the cel::Value APIs. + friend class ProtoMessageTypeAdapter; + + LegacyTypeMutationApis() = default; }; // Interface for access apis. @@ -138,6 +149,14 @@ class LegacyTypeAccessApis { virtual std::vector ListFields( const CelValue::MessageWrapper& instance) const = 0; + + private: + // This class should only be implemented by CEL. Custom structs are only + // supported using the cel::Value APIs. + friend class DucktypedMessageAdapter; + friend class ProtoMessageTypeAdapter; + + LegacyTypeAccessApis() = default; }; // Type information about a legacy Struct type. diff --git a/eval/public/structs/legacy_type_adapter_test.cc b/eval/public/structs/legacy_type_adapter_test.cc index 4c16a59ad..1dabac7d7 100644 --- a/eval/public/structs/legacy_type_adapter_test.cc +++ b/eval/public/structs/legacy_type_adapter_test.cc @@ -14,49 +14,19 @@ #include "eval/public/structs/legacy_type_adapter.h" -#include - -#include "eval/public/cel_value.h" -#include "eval/public/structs/trivial_legacy_type_info.h" -#include "eval/public/testing/matchers.h" +#include "eval/public/structs/proto_message_type_adapter.h" #include "eval/testutil/test_message.pb.h" -#include "extensions/protobuf/memory_manager.h" -#include "internal/status_macros.h" #include "internal/testing.h" namespace google::api::expr::runtime { namespace { -class TestAccessApiImpl : public LegacyTypeAccessApis { - public: - TestAccessApiImpl() {} - absl::StatusOr HasField( - absl::string_view field_name, - const CelValue::MessageWrapper& value) const override { - return absl::UnimplementedError("Not implemented"); - } - - absl::StatusOr GetField( - absl::string_view field_name, const CelValue::MessageWrapper& instance, - ProtoWrapperTypeOptions unboxing_option, - cel::MemoryManagerRef memory_manager) const override { - return absl::UnimplementedError("Not implemented"); - } - - std::vector ListFields( - const CelValue::MessageWrapper& instance) const override { - return std::vector(); - } -}; - -TEST(LegacyTypeAdapterAccessApis, DefaultAlwaysInequal) { - TestMessage message; - MessageWrapper wrapper(&message, nullptr); - MessageWrapper wrapper2(&message, nullptr); - - TestAccessApiImpl impl; +TEST(LegacyTypeAdapter, Basic) { + ProtoMessageTypeAdapter adapter(TestMessage::descriptor(), nullptr); + LegacyTypeAdapter type_adapter(&adapter, &adapter); - EXPECT_FALSE(impl.IsEqualTo(wrapper, wrapper2)); + EXPECT_EQ(type_adapter.access_apis(), &adapter); + EXPECT_EQ(type_adapter.mutation_apis(), &adapter); } } // namespace diff --git a/eval/public/structs/legacy_type_info_apis.h b/eval/public/structs/legacy_type_info_apis.h index e470ac566..90e2f3710 100644 --- a/eval/public/structs/legacy_type_info_apis.h +++ b/eval/public/structs/legacy_type_info_apis.h @@ -15,10 +15,10 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_LEGACY_TYPE_INFO_APIS_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_LEGACY_TYPE_INFO_APIS_H_ +#include #include #include "absl/base/nullability.h" -#include "absl/status/status.h" #include "absl/strings/string_view.h" #include "eval/public/message_wrapper.h" #include "google/protobuf/descriptor.h" @@ -29,6 +29,11 @@ namespace google::api::expr::runtime { class LegacyTypeAccessApis; class LegacyTypeMutationApis; +// Forward declare permitted subclasses. +class DucktypedMessageAdapter; +class ProtoMessageTypeAdapter; +class TrivialTypeInfo; + // Interface for providing type info from a user defined type (represented as a // message). // @@ -97,10 +102,19 @@ class LegacyTypeInfoApis { // // The underlying string is expected to remain valid as long as the // LegacyTypeInfoApis instance. - virtual absl::optional FindFieldByName( + virtual std::optional FindFieldByName( absl::string_view name [[maybe_unused]]) const { - return absl::nullopt; + return std::nullopt; } + + private: + // This class should only be implemented by CEL. Custom structs are only + // supported using the cel::Value APIs. + friend class DucktypedMessageAdapter; + friend class ProtoMessageTypeAdapter; + friend class TrivialTypeInfo; + + LegacyTypeInfoApis() = default; }; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_provider.h b/eval/public/structs/legacy_type_provider.h index e2e67411c..6148caee7 100644 --- a/eval/public/structs/legacy_type_provider.h +++ b/eval/public/structs/legacy_type_provider.h @@ -15,11 +15,12 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TYPE_PROVIDER_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TYPE_PROVIDER_H_ +#include + #include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/type.h" #include "common/type_reflector.h" #include "common/value.h" @@ -45,7 +46,7 @@ class LegacyTypeProvider : public cel::TypeReflector { // Returned non-null pointers from the adapter implemententation must remain // valid as long as the type provider. // TODO(uncreated-issue/3): add alternative for new type system. - virtual absl::optional ProvideLegacyType( + virtual std::optional ProvideLegacyType( absl::string_view name) const = 0; // Return LegacyTypeInfoApis for the fully qualified type name if available. @@ -55,9 +56,9 @@ class LegacyTypeProvider : public cel::TypeReflector { // Since custom type providers should create values compatible with evaluator // created ones, the TypeInfoApis returned from this method should be the same // as the ones used in value creation. - virtual absl::optional ProvideLegacyTypeInfo( + virtual std::optional ProvideLegacyTypeInfo( ABSL_ATTRIBUTE_UNUSED absl::string_view name) const { - return absl::nullopt; + return std::nullopt; } absl::StatusOr NewValueBuilder( @@ -66,10 +67,10 @@ class LegacyTypeProvider : public cel::TypeReflector { google::protobuf::Arena* absl_nonnull arena) const final; protected: - absl::StatusOr> FindTypeImpl( + absl::StatusOr> FindTypeImpl( absl::string_view name) const final; - absl::StatusOr> + absl::StatusOr> FindStructTypeFieldByNameImpl(absl::string_view type, absl::string_view name) const final; }; diff --git a/eval/public/structs/legacy_type_provider_test.cc b/eval/public/structs/legacy_type_provider_test.cc index 2da45e69d..cd683bb50 100644 --- a/eval/public/structs/legacy_type_provider_test.cc +++ b/eval/public/structs/legacy_type_provider_test.cc @@ -15,19 +15,19 @@ #include "eval/public/structs/legacy_type_provider.h" #include -#include -#include "absl/status/status_matchers.h" #include "absl/strings/string_view.h" #include "common/type.h" +#include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" +#include "eval/public/structs/proto_message_type_adapter.h" +#include "eval/public/structs/trivial_legacy_type_info.h" +#include "eval/testutil/test_message.pb.h" #include "internal/testing.h" namespace google::api::expr::runtime { namespace { -using ::absl_testing::IsOk; - class LegacyTypeProviderTestEmpty : public LegacyTypeProvider { public: absl::optional ProvideLegacyType( @@ -36,32 +36,6 @@ class LegacyTypeProviderTestEmpty : public LegacyTypeProvider { } }; -class LegacyTypeInfoApisEmpty : public LegacyTypeInfoApis { - public: - std::string DebugString( - const MessageWrapper& wrapped_message) const override { - return ""; - } - absl::string_view GetTypename( - const MessageWrapper& wrapped_message) const override { - return test_string_; - } - const LegacyTypeAccessApis* GetAccessApis( - const MessageWrapper& wrapped_message) const override { - return nullptr; - } - absl::optional FindFieldByName( - absl::string_view name) const override { - if (name == "field1") { - return FieldDescription{1, "field1"}; - } - return absl::nullopt; - } - - private: - const std::string test_string_ = "test"; -}; - class LegacyTypeProviderTestImpl : public LegacyTypeProvider { public: explicit LegacyTypeProviderTestImpl(const LegacyTypeInfoApis* test_type_info) @@ -92,8 +66,7 @@ TEST(LegacyTypeProviderTest, EmptyTypeProviderHasProvideTypeInfo) { } TEST(LegacyTypeProviderTest, NonEmptyTypeProviderProvidesSomeTypes) { - LegacyTypeInfoApisEmpty test_type_info; - LegacyTypeProviderTestImpl provider(&test_type_info); + LegacyTypeProviderTestImpl provider(TrivialTypeInfo::GetInstance()); EXPECT_TRUE(provider.ProvideLegacyType("test").has_value()); EXPECT_TRUE(provider.ProvideLegacyTypeInfo("test").has_value()); EXPECT_EQ(provider.ProvideLegacyType("other"), std::nullopt); @@ -101,15 +74,16 @@ TEST(LegacyTypeProviderTest, NonEmptyTypeProviderProvidesSomeTypes) { } TEST(LegacyTypeProviderTest, FindStructTypeFieldByName) { - LegacyTypeInfoApisEmpty test_type_info; - LegacyTypeProviderTestImpl provider(&test_type_info); + ProtoMessageTypeAdapter adapter(TestMessage::descriptor(), nullptr); + LegacyTypeProviderTestImpl provider(&adapter); - ASSERT_OK_AND_ASSIGN(absl::optional field, - provider.FindStructTypeFieldByName("test", "field1")); + ASSERT_OK_AND_ASSIGN( + absl::optional field, + provider.FindStructTypeFieldByName("test", "int32_value")); ASSERT_TRUE(field.has_value()); - EXPECT_EQ(field->name(), "field1"); + EXPECT_EQ(field->name(), "int32_value"); EXPECT_EQ(field->number(), 1); - EXPECT_EQ(field->GetType(), cel::DynType()); + EXPECT_EQ(field->GetType(), cel::IntType()); ASSERT_OK_AND_ASSIGN( absl::optional not_found_field, diff --git a/eval/public/structs/proto_message_type_adapter.cc b/eval/public/structs/proto_message_type_adapter.cc index a8bb852fc..db5f3c6e2 100644 --- a/eval/public/structs/proto_message_type_adapter.cc +++ b/eval/public/structs/proto_message_type_adapter.cc @@ -15,7 +15,7 @@ #include "eval/public/structs/proto_message_type_adapter.h" #include -#include +#include #include #include #include @@ -27,7 +27,6 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "base/attribute.h" #include "common/memory.h" @@ -42,7 +41,6 @@ #include "eval/public/structs/legacy_type_info_apis.h" #include "extensions/protobuf/internal/qualify.h" #include "extensions/protobuf/memory_manager.h" -#include "internal/casts.h" #include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -178,7 +176,7 @@ class LegacyQualifyState final LegacyQualifyState(const LegacyQualifyState&) = delete; LegacyQualifyState& operator=(const LegacyQualifyState&) = delete; - absl::optional& result() { return result_; } + std::optional& result() { return result_; } private: void SetResultFromError(absl::Status status, @@ -221,7 +219,7 @@ class LegacyQualifyState final return absl::OkStatus(); } - absl::optional result_; + std::optional result_; }; absl::StatusOr QualifyImpl( @@ -279,8 +277,9 @@ std::vector ListFieldsImpl( return field_names; } +} // namespace + class DucktypedMessageAdapter : public LegacyTypeAccessApis, - public LegacyTypeMutationApis, public LegacyTypeInfoApis { public: // Implement field access APIs. @@ -351,46 +350,6 @@ class DucktypedMessageAdapter : public LegacyTypeAccessApis, return message->ShortDebugString(); } - bool DefinesField(absl::string_view field_name) const override { - // Pretend all our fields exist. Real errors will be returned from field - // getters and setters. - return true; - } - - absl::StatusOr NewInstance( - cel::MemoryManagerRef memory_manager) const override { - return absl::UnimplementedError("NewInstance is not implemented"); - } - - absl::StatusOr AdaptFromWellKnownType( - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder instance) const override { - if (!instance.HasFullProto() || instance.message_ptr() == nullptr) { - return absl::UnimplementedError( - "MessageLite is not supported, descriptor is required"); - } - return ProtoMessageTypeAdapter( - static_cast(instance.message_ptr()) - ->GetDescriptor(), - nullptr) - .AdaptFromWellKnownType(memory_manager, instance); - } - - absl::Status SetField( - absl::string_view field_name, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const override { - if (!instance.HasFullProto() || instance.message_ptr() == nullptr) { - return absl::UnimplementedError( - "MessageLite is not supported, descriptor is required"); - } - return ProtoMessageTypeAdapter( - static_cast(instance.message_ptr()) - ->GetDescriptor(), - nullptr) - .SetField(field_name, value, memory_manager, instance); - } - std::vector ListFields( const CelValue::MessageWrapper& instance) const override { return ListFieldsImpl(instance); @@ -403,7 +362,7 @@ class DucktypedMessageAdapter : public LegacyTypeAccessApis, const LegacyTypeMutationApis* GetMutationApis( const MessageWrapper& wrapped_message) const override { - return this; + return nullptr; } static const DucktypedMessageAdapter& GetSingleton() { @@ -412,6 +371,8 @@ class DucktypedMessageAdapter : public LegacyTypeAccessApis, } }; +namespace { + CelValue MessageCelValueFactory(const google::protobuf::Message* message) { return CelValue::CreateMessageWrapper( MessageWrapper(message, &DucktypedMessageAdapter::GetSingleton())); @@ -469,7 +430,7 @@ const LegacyTypeAccessApis* ProtoMessageTypeAdapter::GetAccessApis( return this; } -absl::optional +std::optional ProtoMessageTypeAdapter::FindFieldByName(absl::string_view field_name) const { if (descriptor_ == nullptr) { return std::nullopt; diff --git a/extensions/BUILD b/extensions/BUILD index a60943f61..faf9f08c8 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -377,7 +377,8 @@ cc_test( "//common:decl_proto", "//common:expr", "//common:kind", - "//common:memory", + "//common:native_type", + "//common:typeinfo", "//common:value", "//compiler", "//compiler:compiler_factory", @@ -391,8 +392,6 @@ cc_test( "//eval/public:cel_type_registry", "//eval/public:cel_value", "//eval/public/structs:cel_proto_wrapper", - "//eval/public/structs:legacy_type_adapter", - "//eval/public/structs:legacy_type_info_apis", "//extensions/protobuf:ast_converters", "//internal:number", "//internal:status_macros", @@ -415,7 +414,6 @@ cc_test( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", diff --git a/extensions/select_optimization_test.cc b/extensions/select_optimization_test.cc index c14c4d461..27c191738 100644 --- a/extensions/select_optimization_test.cc +++ b/extensions/select_optimization_test.cc @@ -33,7 +33,6 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "base/ast.h" #include "base/attribute.h" @@ -45,8 +44,10 @@ #include "common/decl_proto.h" #include "common/expr.h" #include "common/kind.h" -#include "common/memory.h" +#include "common/native_type.h" +#include "common/typeinfo.h" #include "common/value.h" +#include "common/values/custom_struct_value.h" #include "compiler/compiler.h" #include "compiler/compiler_factory.h" #include "compiler/optional.h" @@ -59,8 +60,6 @@ #include "eval/public/cel_type_registry.h" #include "eval/public/cel_value.h" #include "eval/public/structs/cel_proto_wrapper.h" -#include "eval/public/structs/legacy_type_adapter.h" -#include "eval/public/structs/legacy_type_info_apis.h" #include "extensions/protobuf/ast_converters.h" #include "internal/number.h" #include "internal/status_macros.h" @@ -96,18 +95,10 @@ using ::google::api::expr::runtime::CelProtoWrapper; using ::google::api::expr::runtime::CelValue; using ::google::api::expr::runtime::FlatExprBuilder; using ::google::api::expr::runtime::FlatExpression; -using ::google::api::expr::runtime::LegacyTypeAccessApis; -using ::google::api::expr::runtime::LegacyTypeInfoApis; -using ::google::api::expr::runtime::LegacyTypeMutationApis; -using ::google::protobuf::Empty; -using ::testing::_; using ::testing::AllOf; -using ::testing::AnyOf; using ::testing::ElementsAre; using ::testing::Eq; using ::testing::HasSubstr; -using ::testing::NiceMock; -using ::testing::Return; using ::testing::SizeIs; using ::testing::Truly; @@ -232,75 +223,105 @@ absl::StatusOr> CompileForTestCase( return r.ReleaseAst(); } -class MockAccessApis : public LegacyTypeInfoApis, public LegacyTypeAccessApis { +class TestPartialQualifyStruct : public CustomStructValueInterface { public: - std::string DebugString( - const MessageWrapper& wrapped_message) const override { - return "MockAccessApis"; + explicit TestPartialQualifyStruct(Value leaf_value) + : leaf_value_(std::move(leaf_value)) {} + + std::string DebugString() const override { + return "TestPartialQualifyStruct"; + } + + absl::Status SerializeTo( + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::io::ZeroCopyOutputStream* output) const override { + return absl::UnimplementedError("SerializeTo"); } - absl::string_view GetTypename( - const MessageWrapper& wrapped_message) const override { - return "MockAccessApis"; + absl::Status ConvertToJsonObject( + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Message* json) const override { + return absl::UnimplementedError("ConvertToJsonObject"); } - const LegacyTypeAccessApis* GetAccessApis( - const MessageWrapper& wrapped_message) const override { - return this; + absl::string_view GetTypeName() const override { + return "cel.expr.conformance.proto2.NestedTestAllTypes"; } - const LegacyTypeMutationApis* GetMutationApis( - const MessageWrapper& wrapped_message) const override { - return nullptr; + bool IsZeroValue() const override { return false; } + + absl::Status GetFieldByName(absl::string_view name, + ProtoWrapperTypeOptions unboxing_options, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, + Value* result) const override { + if (name == "child" || name == "payload" || name == "standalone_message") { + *result = CustomStructValue(this, arena); + return absl::OkStatus(); + } + if (name == "bb") { + *result = leaf_value_; + return absl::OkStatus(); + } + return NoSuchFieldError(name).ToStatus(); } - std::optional< - google::api::expr::runtime::LegacyTypeInfoApis::FieldDescription> - FindFieldByName(absl::string_view field_name) const override { - return std::nullopt; + absl::Status GetFieldByNumber(int64_t number, + ProtoWrapperTypeOptions unboxing_options, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, + Value* result) const override { + return absl::UnimplementedError("GetFieldByNumber"); } - MOCK_METHOD(absl::StatusOr, GetField, - (absl::string_view field_name, - const CelValue::MessageWrapper& instance, - ProtoWrapperTypeOptions unboxing_option, - cel::MemoryManagerRef memory_manager), - (const, override)); - - MOCK_METHOD(absl::StatusOr, HasField, - (absl::string_view field_name, - const CelValue::MessageWrapper& value), - (const, override)); - - MOCK_METHOD(absl::StatusOr, - Qualify, - (absl::Span qualifiers, - const CelValue::MessageWrapper& instance, bool presence_test, - MemoryManagerRef memory_manager), - (const, override)); - - bool IsEqualTo( - const CelValue::MessageWrapper& instance, - const CelValue::MessageWrapper& other_instance) const override { + absl::StatusOr HasFieldByName(absl::string_view name) const override { + if (name == "child" || name == "payload" || name == "standalone_message" || + name == "bb") { + return true; + } return false; } - std::vector ListFields( - const CelValue::MessageWrapper& instance) const override { - return {}; + absl::StatusOr HasFieldByNumber(int64_t number) const override { + return absl::UnimplementedError("HasFieldByNumber"); } -}; -std::pair MakeMockLegacyMessage( - google::protobuf::Arena* arena) { - auto* mock_access_apis = - google::protobuf::Arena::Create>(arena); - auto* message = google::protobuf::Arena::Create(arena); + absl::Status ForEachField(ForEachFieldCallback callback, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena) const override { + return absl::OkStatus(); + } - CelValue::MessageWrapper::Builder wrapper(message); - return {mock_access_apis, - CelValue::CreateMessageWrapper(wrapper.Build(mock_access_apis))}; -} + absl::Status Qualify(absl::Span qualifiers, + bool presence_test, + const google::protobuf::DescriptorPool* descriptor_pool, + google::protobuf::MessageFactory* message_factory, + google::protobuf::Arena* arena, Value* result, + int* count) const override { + if (qualifiers.size() >= 3) { + *count = 3; + *result = CustomStructValue(this, arena); + return absl::OkStatus(); + } + return absl::UnimplementedError("Qualify unsupported"); + } + + CustomStructValue Clone(google::protobuf::Arena* arena) const override { + return CustomStructValue(this, arena); + } + + NativeTypeId GetNativeTypeId() const override { + return cel::TypeId(); + } + + private: + Value leaf_value_; +}; absl::Status TestBindLegacyValue(absl::string_view variable, CelValue legacy_value, google::protobuf::Arena* arena, @@ -1031,24 +1052,12 @@ INSTANTIATE_TEST_SUITE_P( "nested_test_all_types.child.payload.standalone_message.bb", {}, [](google::protobuf::Arena* arena, Activation& act) { - auto mock_pair = MakeMockLegacyMessage(arena); - MockAccessApis* mock = mock_pair.first; - CelValue mocked_value = mock_pair.second; - ON_CALL(*mock, Qualify(SizeIs(4), _, /*presence_test=*/false, _)) - .WillByDefault( - Return(LegacyTypeAccessApis::LegacyQualifyResult{ - mocked_value, 3})); - ON_CALL(*mock, GetField("bb", _, _, _)) - .WillByDefault(Return(CelValue::CreateInt64(42))); - - // Support the forced-fallback case. - ON_CALL(*mock, GetField(AnyOf(Eq("child"), Eq("payload"), - Eq("standalone_message")), - _, _, _)) - .WillByDefault(Return(mocked_value)); - - return TestBindLegacyValue("nested_test_all_types", mocked_value, - arena, act); + auto* custom_struct = + google::protobuf::Arena::Create(arena, + IntValue(42)); + act.InsertOrAssignValue("nested_test_all_types", + CustomStructValue(custom_struct, arena)); + return absl::OkStatus(); }, [](const absl::StatusOr& got) { ASSERT_OK_AND_ASSIGN(Value result, got); @@ -1061,29 +1070,12 @@ INSTANTIATE_TEST_SUITE_P( "has(nested_test_all_types.child.payload.standalone_message.bb)", {}, [](google::protobuf::Arena* arena, Activation& act) { - auto mock_pair = MakeMockLegacyMessage(arena); - MockAccessApis* mock = mock_pair.first; - CelValue mocked_value = mock_pair.second; - ON_CALL(*mock, Qualify(SizeIs(4), _, /*presence_test=*/true, _)) - .WillByDefault( - Return(LegacyTypeAccessApis::LegacyQualifyResult{ - mocked_value, 3})); - ON_CALL(*mock, HasField("bb", _)).WillByDefault(Return(true)); - ON_CALL(*mock, GetField("bb", _, _, _)) - .WillByDefault(Return(CelValue::CreateInt64(42))); - - // Support the forced-fallback case. - ON_CALL(*mock, GetField(AnyOf(Eq("child"), Eq("payload"), - Eq("standalone_message")), - _, _, _)) - .WillByDefault(Return(mocked_value)); - ON_CALL(*mock, HasField(AnyOf(Eq("child"), Eq("payload"), - Eq("standalone_message")), - _)) - .WillByDefault(Return(true)); - - return TestBindLegacyValue("nested_test_all_types", mocked_value, - arena, act); + auto* custom_struct = + google::protobuf::Arena::Create(arena, + IntValue(42)); + act.InsertOrAssignValue("nested_test_all_types", + CustomStructValue(custom_struct, arena)); + return absl::OkStatus(); }, [](const absl::StatusOr& got) { ASSERT_OK_AND_ASSIGN(Value result, got); From 3ad5ba84c57a789b8c6ca242c382edb6e1003d1a Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 5 Aug 2026 12:36:54 -0700 Subject: [PATCH 166/350] Decouple LegacyTypeProvider and cel::TypeProvider Move the implementation to the concrete / internal class LegacyRuntimeTypeProvider. PiperOrigin-RevId: 959810409 --- eval/public/structs/BUILD | 22 +--- eval/public/structs/legacy_type_provider.h | 31 +---- .../structs/legacy_type_provider_test.cc | 22 +--- .../structs/proto_message_type_adapter.h | 9 +- .../protobuf_descriptor_type_provider_test.cc | 60 --------- runtime/internal/BUILD | 33 +++++ .../internal/legacy_runtime_type_provider.cc | 106 +++++---------- .../internal/legacy_runtime_type_provider.h | 23 +++- .../legacy_runtime_type_provider_test.cc | 121 ++++++++++++++++++ 9 files changed, 222 insertions(+), 205 deletions(-) rename eval/public/structs/legacy_type_provider.cc => runtime/internal/legacy_runtime_type_provider.cc (62%) create mode 100644 runtime/internal/legacy_runtime_type_provider_test.cc diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 75b4685a0..3ffe5994c 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -229,26 +229,12 @@ cc_test( cc_library( name = "legacy_type_provider", - srcs = ["legacy_type_provider.cc"], hdrs = ["legacy_type_provider.h"], deps = [ ":legacy_type_adapter", ":legacy_type_info_apis", - "//common:legacy_value", - "//common:memory", - "//common:type", - "//common:value", - "//eval/public:message_wrapper", - "//extensions/protobuf:memory_manager", - "//internal:status_macros", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", - "@com_google_absl//absl/types:optional", - "@com_google_protobuf//:protobuf", ], ) @@ -359,15 +345,11 @@ cc_test( deps = [ ":legacy_type_info_apis", ":protobuf_descriptor_type_provider", - "//common:type", "//eval/public:cel_value", "//eval/public/testing:matchers", "//extensions/protobuf:memory_manager", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", "@com_google_absl//absl/status:status_matchers", - "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:wrappers_cc_proto", ], @@ -413,12 +395,10 @@ cc_test( ":legacy_type_adapter", ":legacy_type_info_apis", ":legacy_type_provider", - ":proto_message_type_adapter", ":trivial_legacy_type_info", - "//common:type", - "//eval/testutil:test_message_cc_proto", "//internal:testing", "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", ], ) diff --git a/eval/public/structs/legacy_type_provider.h b/eval/public/structs/legacy_type_provider.h index 6148caee7..b00645d0e 100644 --- a/eval/public/structs/legacy_type_provider.h +++ b/eval/public/structs/legacy_type_provider.h @@ -18,24 +18,20 @@ #include #include "absl/base/attributes.h" -#include "absl/base/nullability.h" -#include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "common/type.h" -#include "common/type_reflector.h" -#include "common/value.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/message.h" namespace google::api::expr::runtime { -// An internal extension of cel::TypeProvider that also deals with legacy types. +// Provides legacy type adapters for google::api::expr::runtime::CelValue +// wrapping messages. // -// Note: This API is not finalized. Consult the CEL team before introducing new -// implementations. -class LegacyTypeProvider : public cel::TypeReflector { +// Note: internal use of this type is limited to interop with the Legacy +// (CelValue) runtime with an internal implementation. +// +// New implementations cannot be registered with the evaluator. +class LegacyTypeProvider { public: virtual ~LegacyTypeProvider() = default; @@ -60,19 +56,6 @@ class LegacyTypeProvider : public cel::TypeReflector { ABSL_ATTRIBUTE_UNUSED absl::string_view name) const { return std::nullopt; } - - absl::StatusOr NewValueBuilder( - absl::string_view name, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) const final; - - protected: - absl::StatusOr> FindTypeImpl( - absl::string_view name) const final; - - absl::StatusOr> - FindStructTypeFieldByNameImpl(absl::string_view type, - absl::string_view name) const final; }; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_provider_test.cc b/eval/public/structs/legacy_type_provider_test.cc index cd683bb50..c8059da7c 100644 --- a/eval/public/structs/legacy_type_provider_test.cc +++ b/eval/public/structs/legacy_type_provider_test.cc @@ -17,12 +17,10 @@ #include #include "absl/strings/string_view.h" -#include "common/type.h" +#include "absl/types/optional.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" -#include "eval/public/structs/proto_message_type_adapter.h" #include "eval/public/structs/trivial_legacy_type_info.h" -#include "eval/testutil/test_message.pb.h" #include "internal/testing.h" namespace google::api::expr::runtime { @@ -73,23 +71,5 @@ TEST(LegacyTypeProviderTest, NonEmptyTypeProviderProvidesSomeTypes) { EXPECT_EQ(provider.ProvideLegacyTypeInfo("other"), std::nullopt); } -TEST(LegacyTypeProviderTest, FindStructTypeFieldByName) { - ProtoMessageTypeAdapter adapter(TestMessage::descriptor(), nullptr); - LegacyTypeProviderTestImpl provider(&adapter); - - ASSERT_OK_AND_ASSIGN( - absl::optional field, - provider.FindStructTypeFieldByName("test", "int32_value")); - ASSERT_TRUE(field.has_value()); - EXPECT_EQ(field->name(), "int32_value"); - EXPECT_EQ(field->number(), 1); - EXPECT_EQ(field->GetType(), cel::IntType()); - - ASSERT_OK_AND_ASSIGN( - absl::optional not_found_field, - provider.FindStructTypeFieldByName("test", "unknown_field")); - EXPECT_FALSE(not_found_field.has_value()); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/public/structs/proto_message_type_adapter.h b/eval/public/structs/proto_message_type_adapter.h index eb89a467c..f4e6b4c8b 100644 --- a/eval/public/structs/proto_message_type_adapter.h +++ b/eval/public/structs/proto_message_type_adapter.h @@ -28,6 +28,7 @@ #include "eval/public/structs/legacy_type_info_apis.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" namespace google::api::expr::runtime { @@ -40,7 +41,7 @@ class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, public LegacyTypeAccessApis, public LegacyTypeMutationApis { public: - ProtoMessageTypeAdapter(const google::protobuf::Descriptor* descriptor, + ProtoMessageTypeAdapter(const google::protobuf::Descriptor* absl_nonnull descriptor, google::protobuf::MessageFactory* message_factory) : message_factory_(message_factory), descriptor_(descriptor) {} @@ -107,6 +108,10 @@ class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, std::vector ListFields( const CelValue::MessageWrapper& instance) const override; + const google::protobuf::Descriptor* absl_nonnull descriptor() const { + return descriptor_; + } + private: // Helper for standardizing error messages for SetField operation. absl::Status ValidateSetFieldOp(bool assertion, absl::string_view field, @@ -117,7 +122,7 @@ class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, google::protobuf::Message* message) const; google::protobuf::MessageFactory* message_factory_; - const google::protobuf::Descriptor* descriptor_; + const google::protobuf::Descriptor* absl_nonnull descriptor_; }; // Creates a CelValue from the given field on the proto message. This is the diff --git a/eval/public/structs/protobuf_descriptor_type_provider_test.cc b/eval/public/structs/protobuf_descriptor_type_provider_test.cc index 1d07fd657..f2b7900ef 100644 --- a/eval/public/structs/protobuf_descriptor_type_provider_test.cc +++ b/eval/public/structs/protobuf_descriptor_type_provider_test.cc @@ -18,15 +18,11 @@ #include "google/protobuf/wrappers.pb.h" #include "absl/status/status_matchers.h" -#include "common/type.h" #include "eval/public/cel_value.h" #include "eval/public/structs/legacy_type_info_apis.h" #include "eval/public/testing/matchers.h" #include "extensions/protobuf/memory_manager.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -35,7 +31,6 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::cel::expr::conformance::proto3::TestAllTypes; using ::cel::extensions::ProtoMemoryManager; TEST(ProtobufDescriptorProvider, Basic) { @@ -101,60 +96,5 @@ TEST(ProtobufDescriptorProvider, NotFound) { ASSERT_FALSE(type_info.has_value()); } -TEST(ProtobufDescriptorProvider, FindType) { - ProtobufDescriptorProvider provider( - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory()); - ASSERT_OK_AND_ASSIGN(std::optional wrapper_type, - provider.FindType("google.protobuf.Int64Value")); - ASSERT_TRUE(wrapper_type.has_value()); - EXPECT_TRUE(wrapper_type->Is()); - EXPECT_EQ(wrapper_type->name(), "google.protobuf.Int64Value"); - - ASSERT_OK_AND_ASSIGN( - std::optional msg_type, - provider.FindType("cel.expr.conformance.proto3.TestAllTypes")); - ASSERT_TRUE(msg_type.has_value()); - EXPECT_TRUE(msg_type->Is()); - EXPECT_EQ(msg_type->name(), "cel.expr.conformance.proto3.TestAllTypes"); -} - -TEST(ProtobufDescriptorProvider, FindStructTypeFieldByName) { - ProtobufDescriptorProvider provider( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); - ASSERT_OK_AND_ASSIGN(std::optional field, - provider.FindStructTypeFieldByName( - "google.protobuf.Int64Value", "value")); - ASSERT_TRUE(field.has_value()); - EXPECT_EQ(field->name(), "value"); - EXPECT_EQ(field->number(), 1); - EXPECT_EQ(field->GetType(), cel::IntType()); -} - -TEST(ProtobufDescriptorProvider, FindTypeNotFound) { - ProtobufDescriptorProvider provider( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); - ASSERT_OK_AND_ASSIGN(std::optional type, - provider.FindType("UnknownType")); - EXPECT_FALSE(type.has_value()); -} - -TEST(ProtobufDescriptorProvider, FindStructTypeFieldByNameNotFound) { - ProtobufDescriptorProvider provider( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); - ASSERT_OK_AND_ASSIGN(std::optional field, - provider.FindStructTypeFieldByName( - "google.protobuf.Int64Value", "unknown_field")); - EXPECT_FALSE(field.has_value()); - - ASSERT_OK_AND_ASSIGN( - std::optional field2, - provider.FindStructTypeFieldByName("UnknownType", "value")); - EXPECT_FALSE(field2.has_value()); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index 6c96de1cf..44671102a 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -181,10 +181,43 @@ cc_library( cc_library( name = "legacy_runtime_type_provider", + srcs = ["legacy_runtime_type_provider.cc"], hdrs = ["legacy_runtime_type_provider.h"], deps = [ + "//common:legacy_value", + "//common:memory", + "//common:type", + "//common:value", + "//eval/public:message_wrapper", + "//eval/public/structs:legacy_type_adapter", + "//eval/public/structs:legacy_type_info_apis", + "//eval/public/structs:proto_message_type_adapter", "//eval/public/structs:protobuf_descriptor_type_provider", + "//extensions/protobuf:memory_manager", + "//internal:status_macros", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:optional", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "legacy_runtime_type_provider_test", + srcs = ["legacy_runtime_type_provider_test.cc"], + deps = [ + ":legacy_runtime_type_provider", + "//common:type", + "//common:value", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//internal:testing_message_factory", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/types:optional", + "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", ], ) diff --git a/eval/public/structs/legacy_type_provider.cc b/runtime/internal/legacy_runtime_type_provider.cc similarity index 62% rename from eval/public/structs/legacy_type_provider.cc rename to runtime/internal/legacy_runtime_type_provider.cc index cd7079451..5db0b448e 100644 --- a/eval/public/structs/legacy_type_provider.cc +++ b/runtime/internal/legacy_runtime_type_provider.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "eval/public/structs/legacy_type_provider.h" +#include "runtime/internal/legacy_runtime_type_provider.h" #include #include @@ -33,73 +33,21 @@ #include "eval/public/message_wrapper.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" +#include "eval/public/structs/proto_message_type_adapter.h" #include "extensions/protobuf/memory_manager.h" #include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" -namespace google::api::expr::runtime { +namespace cel::runtime_internal { namespace { using google::api::expr::runtime::LegacyTypeAdapter; +using google::api::expr::runtime::LegacyTypeInfoApis; using google::api::expr::runtime::MessageWrapper; -class LegacyStructValueBuilder final : public cel::StructValueBuilder { - public: - LegacyStructValueBuilder(cel::MemoryManagerRef memory_manager, - LegacyTypeAdapter adapter, - MessageWrapper::Builder builder) - : memory_manager_(memory_manager), - adapter_(adapter), - builder_(std::move(builder)) {} - - absl::StatusOr> SetFieldByName( - absl::string_view name, cel::Value value) override { - CEL_ASSIGN_OR_RETURN( - auto legacy_value, - LegacyValue(cel::extensions::ProtoMemoryManagerArena(memory_manager_), - value), - _.With(cel::ErrorValueReturn())); - CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetField( - name, legacy_value, memory_manager_, builder_)) - .With(cel::ErrorValueReturn()); - return std::nullopt; - } - - absl::StatusOr> SetFieldByNumber( - int64_t number, cel::Value value) override { - CEL_ASSIGN_OR_RETURN( - auto legacy_value, - LegacyValue(cel::extensions::ProtoMemoryManagerArena(memory_manager_), - value), - _.With(cel::ErrorValueReturn())); - CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetFieldByNumber( - number, legacy_value, memory_manager_, builder_)) - .With(cel::ErrorValueReturn()); - return std::nullopt; - } - - absl::StatusOr Build() && override { - CEL_ASSIGN_OR_RETURN(auto message, - adapter_.mutation_apis()->AdaptFromWellKnownType( - memory_manager_, std::move(builder_))); - if (!message.IsMessage()) { - return absl::FailedPreconditionError("expected MessageWrapper"); - } - auto message_wrapper = message.MessageWrapperOrDie(); - return cel::common_internal::LegacyStructValue( - google::protobuf::DownCastMessage(message_wrapper.message_ptr()), - message_wrapper.legacy_type_info()); - } - - private: - cel::MemoryManagerRef memory_manager_; - LegacyTypeAdapter adapter_; - MessageWrapper::Builder builder_; -}; - class LegacyValueBuilder final : public cel::ValueBuilder { public: LegacyValueBuilder(cel::MemoryManagerRef memory_manager, @@ -155,28 +103,34 @@ class LegacyValueBuilder final : public cel::ValueBuilder { } // namespace -absl::StatusOr -LegacyTypeProvider::NewValueBuilder( +absl::StatusOr +LegacyRuntimeTypeProvider::NewValueBuilder( absl::string_view name, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { - if (auto type_adapter = ProvideLegacyType(name); type_adapter.has_value()) { - const auto* mutation_apis = type_adapter->mutation_apis(); - if (mutation_apis == nullptr) { - return absl::FailedPreconditionError( - absl::StrCat("LegacyTypeMutationApis missing for type: ", name)); - } - CEL_ASSIGN_OR_RETURN( - auto builder, - mutation_apis->NewInstance(cel::MemoryManagerRef::Pooling(arena))); - return std::make_unique( - cel::MemoryManagerRef::Pooling(arena), *type_adapter, - std::move(builder)); + auto type_adapter = ProvideLegacyType(name); + + if (!type_adapter.has_value()) { + return nullptr; } - return nullptr; + + // We know the implementation should not do this, but can't prove it to type + // system. + // Defensive checks but impractical to exercise. + const auto* mutation_apis = type_adapter->mutation_apis(); + if (mutation_apis == nullptr) { + return absl::FailedPreconditionError( + absl::StrCat("LegacyTypeMutationApis missing for type: ", name)); + } + + CEL_ASSIGN_OR_RETURN( + auto builder, + mutation_apis->NewInstance(cel::MemoryManagerRef::Pooling(arena))); + return std::make_unique( + cel::MemoryManagerRef::Pooling(arena), *type_adapter, std::move(builder)); } -absl::StatusOr> LegacyTypeProvider::FindTypeImpl( +absl::StatusOr> LegacyRuntimeTypeProvider::FindTypeImpl( absl::string_view name) const { if (auto type = cel::FindWellKnownType(name); type.has_value()) { return type; @@ -192,14 +146,14 @@ absl::StatusOr> LegacyTypeProvider::FindTypeImpl( return std::nullopt; } -absl::StatusOr> -LegacyTypeProvider::FindStructTypeFieldByNameImpl( +absl::StatusOr> +LegacyRuntimeTypeProvider::FindStructTypeFieldByNameImpl( absl::string_view type, absl::string_view name) const { if (auto result = cel::FindWellKnownTypeFieldByName(type, name); result.has_value()) { return result; } - absl::optional type_info = + std::optional type_info = ProvideLegacyTypeInfo(type); if (!type_info.has_value()) { return std::nullopt; @@ -229,4 +183,4 @@ LegacyTypeProvider::FindStructTypeFieldByNameImpl( return cel::common_internal::BasicStructTypeField(name, 0, cel::DynType{}); } -} // namespace google::api::expr::runtime +} // namespace cel::runtime_internal diff --git a/runtime/internal/legacy_runtime_type_provider.h b/runtime/internal/legacy_runtime_type_provider.h index 503a79b46..45aef5d75 100644 --- a/runtime/internal/legacy_runtime_type_provider.h +++ b/runtime/internal/legacy_runtime_type_provider.h @@ -15,21 +15,42 @@ #ifndef THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_LEGACY_RUNTIME_TYPE_PROVIDER_H_ #define THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_LEGACY_RUNTIME_TYPE_PROVIDER_H_ +#include + #include "absl/base/nullability.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/type.h" +#include "common/type_reflector.h" +#include "common/value.h" #include "eval/public/structs/protobuf_descriptor_type_provider.h" +#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace cel::runtime_internal { class LegacyRuntimeTypeProvider final - : public google::api::expr::runtime::ProtobufDescriptorProvider { + : public google::api::expr::runtime::ProtobufDescriptorProvider, + public TypeReflector { public: LegacyRuntimeTypeProvider( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory) : google::api::expr::runtime::ProtobufDescriptorProvider( descriptor_pool, message_factory) {} + + absl::StatusOr NewValueBuilder( + absl::string_view name, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) const override; + + protected: + absl::StatusOr> FindTypeImpl( + absl::string_view name) const override; + + absl::StatusOr> FindStructTypeFieldByNameImpl( + absl::string_view type, absl::string_view name) const override; }; } // namespace cel::runtime_internal diff --git a/runtime/internal/legacy_runtime_type_provider_test.cc b/runtime/internal/legacy_runtime_type_provider_test.cc new file mode 100644 index 000000000..eceac65a9 --- /dev/null +++ b/runtime/internal/legacy_runtime_type_provider_test.cc @@ -0,0 +1,121 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "runtime/internal/legacy_runtime_type_provider.h" + +#include +#include + +#include "common/type.h" +#include "common/value.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" +#include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace cel::runtime_internal { +namespace { + +using ::cel::expr::conformance::proto3::TestAllTypes; + +TEST(LegacyRuntimeTypeProviderTest, FindType) { + LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory()); + ASSERT_OK_AND_ASSIGN(std::optional wrapper_type, + provider.FindType("google.protobuf.Int64Value")); + ASSERT_TRUE(wrapper_type.has_value()); + EXPECT_TRUE(wrapper_type->Is()); + EXPECT_EQ(wrapper_type->name(), "google.protobuf.Int64Value"); + + ASSERT_OK_AND_ASSIGN( + std::optional msg_type, + provider.FindType("cel.expr.conformance.proto3.TestAllTypes")); + ASSERT_TRUE(msg_type.has_value()); + EXPECT_TRUE(msg_type->Is()); + EXPECT_EQ(msg_type->name(), "cel.expr.conformance.proto3.TestAllTypes"); +} + +TEST(LegacyRuntimeTypeProviderTest, FindTypeNotFound) { + LegacyRuntimeTypeProvider provider( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory()); + ASSERT_OK_AND_ASSIGN(std::optional type, + provider.FindType("UnknownType")); + EXPECT_FALSE(type.has_value()); +} + +TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByName) { + LegacyRuntimeTypeProvider provider( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory()); + ASSERT_OK_AND_ASSIGN(std::optional field, + provider.FindStructTypeFieldByName( + "google.protobuf.Int64Value", "value")); + ASSERT_TRUE(field.has_value()); + EXPECT_EQ(field->name(), "value"); + EXPECT_EQ(field->number(), 1); + EXPECT_EQ(field->GetType(), IntType()); +} + +TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByNameNotFound) { + LegacyRuntimeTypeProvider provider( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory()); + ASSERT_OK_AND_ASSIGN(std::optional field, + provider.FindStructTypeFieldByName( + "google.protobuf.Int64Value", "unknown_field")); + EXPECT_FALSE(field.has_value()); + + ASSERT_OK_AND_ASSIGN( + std::optional field2, + provider.FindStructTypeFieldByName("UnknownType", "value")); + EXPECT_FALSE(field2.has_value()); +} + +TEST(LegacyRuntimeTypeProviderTest, NewValueBuilder) { + LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory()); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(auto builder, + provider.NewValueBuilder( + "cel.expr.conformance.proto3.TestAllTypes", + cel::internal::GetTestingMessageFactory(), &arena)); + ASSERT_NE(builder, nullptr); + + ASSERT_OK_AND_ASSIGN(auto field_result, + builder->SetFieldByName("single_int64", IntValue(42))); + EXPECT_FALSE(field_result.has_value()); + + ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); + EXPECT_TRUE(value.Is()); +} + +TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderNotFound) { + LegacyRuntimeTypeProvider provider( + google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory()); + google::protobuf::LinkMessageReflection(); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN( + auto builder, + provider.NewValueBuilder( + "UnknownType", google::protobuf::MessageFactory::generated_factory(), &arena)); + EXPECT_EQ(builder, nullptr); +} + +} // namespace +} // namespace cel::runtime_internal From 6f716a510d9763b83c8f4f6d6fe62cf3a7761cf5 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 5 Aug 2026 13:37:26 -0700 Subject: [PATCH 167/350] internal updates PiperOrigin-RevId: 959842672 --- tools/testdata/map.textproto | 167 +++++++++++++++++++++++ tools/testdata/map_renumbered.textproto | 168 ++++++++++++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 tools/testdata/map.textproto create mode 100644 tools/testdata/map_renumbered.textproto diff --git a/tools/testdata/map.textproto b/tools/testdata/map.textproto new file mode 100644 index 000000000..f20523d4f --- /dev/null +++ b/tools/testdata/map.textproto @@ -0,0 +1,167 @@ +# proto-file: google/api/expr/checked.proto +# proto-message: CheckedExpr +# has({"key 1": "value 1", "key 2": "value 2", "example_string": "value 3"}.example_string) +type_map { + key: 2 + value { + map_type { + key_type { + primitive: STRING + } + value_type { + primitive: STRING + } + } + } +} +type_map { + key: 4 + value { + primitive: STRING + } +} +type_map { + key: 5 + value { + primitive: STRING + } +} +type_map { + key: 7 + value { + primitive: STRING + } +} +type_map { + key: 8 + value { + primitive: STRING + } +} +type_map { + key: 10 + value { + primitive: STRING + } +} +type_map { + key: 11 + value { + primitive: STRING + } +} +type_map { + key: 13 + value { + primitive: BOOL + } +} +expr { + id: 13 + select_expr { + operand { + id: 2 + struct_expr { + entries { + id: 3 + map_key { + id: 4 + const_expr { + string_value: "key 1" + } + } + value { + id: 5 + const_expr { + string_value: "value 1" + } + } + } + entries { + id: 6 + map_key { + id: 7 + const_expr { + string_value: "key 2" + } + } + value { + id: 8 + const_expr { + string_value: "value 2" + } + } + } + entries { + id: 9 + map_key { + id: 10 + const_expr { + string_value: "example_string" + } + } + value { + id: 11 + const_expr { + string_value: "value 3" + } + } + } + } + } + field: "example_string" + test_only: true + } +} +source_info { + location: "" + line_offsets: 90 + positions { + key: 2 + value: 4 + } + positions { + key: 3 + value: 12 + } + positions { + key: 4 + value: 5 + } + positions { + key: 5 + value: 14 + } + positions { + key: 6 + value: 32 + } + positions { + key: 7 + value: 25 + } + positions { + key: 8 + value: 34 + } + positions { + key: 9 + value: 61 + } + positions { + key: 10 + value: 45 + } + positions { + key: 11 + value: 63 + } + positions { + key: 12 + value: 73 + } + positions { + key: 13 + value: 3 + } +} diff --git a/tools/testdata/map_renumbered.textproto b/tools/testdata/map_renumbered.textproto new file mode 100644 index 000000000..905fe130a --- /dev/null +++ b/tools/testdata/map_renumbered.textproto @@ -0,0 +1,168 @@ +# proto-file: google/api/expr/checked.proto +# proto-message: CheckedExpr +# Renumbered version of map starting from 52. +# has({"key 1": "value 1", "key 2": "value 2", "example_string": "value 3"}.example_string) +type_map { + key: 52 + value { + primitive: STRING + } +} +type_map { + key: 53 + value { + primitive: STRING + } +} +type_map { + key: 54 + value { + primitive: STRING + } +} +type_map { + key: 55 + value { + primitive: STRING + } +} +type_map { + key: 56 + value { + primitive: STRING + } +} +type_map { + key: 57 + value { + primitive: STRING + } +} +type_map { + key: 58 + value { + map_type { + key_type { + primitive: STRING + } + value_type { + primitive: STRING + } + } + } +} +type_map { + key: 62 + value { + primitive: BOOL + } +} +expr { + id: 62 + select_expr { + operand { + id: 58 + struct_expr { + entries { + id: 59 + map_key { + id: 52 + const_expr { + string_value: "key 1" + } + } + value { + id: 53 + const_expr { + string_value: "value 1" + } + } + } + entries { + id: 60 + map_key { + id: 54 + const_expr { + string_value: "key 2" + } + } + value { + id: 55 + const_expr { + string_value: "value 2" + } + } + } + entries { + id: 61 + map_key { + id: 56 + const_expr { + string_value: "example_string" + } + } + value { + id: 57 + const_expr { + string_value: "value 3" + } + } + } + } + } + field: "example_string" + test_only: true + } +} +source_info { + location: "" + line_offsets: 90 + positions { + key: 52 + value: 5 + } + positions { + key: 53 + value: 14 + } + positions { + key: 54 + value: 25 + } + positions { + key: 55 + value: 34 + } + positions { + key: 56 + value: 45 + } + positions { + key: 57 + value: 63 + } + positions { + key: 58 + value: 4 + } + positions { + key: 59 + value: 12 + } + positions { + key: 60 + value: 32 + } + positions { + key: 61 + value: 61 + } + positions { + key: 62 + value: 3 + } + positions { + key: 63 + value: 73 + } +} From a2265a873fe8e5cc3505b182264775a9db9743e6 Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Wed, 5 Aug 2026 15:15:49 -0700 Subject: [PATCH 168/350] Updated MessageToJson conversion for FieldMask in CEL C++ PiperOrigin-RevId: 959896077 --- common/BUILD | 1 + common/values/parsed_message_value.cc | 9 +- common/values/parsed_message_value_test.cc | 14 ++ conformance/BUILD | 2 - eval/public/structs/BUILD | 4 + eval/public/structs/cel_proto_wrap_util.cc | 70 ++++++++- .../structs/cel_proto_wrap_util_test.cc | 142 ++++++++++++++++-- eval/public/structs/cel_proto_wrapper_test.cc | 72 +++++++-- 8 files changed, 282 insertions(+), 32 deletions(-) diff --git a/common/BUILD b/common/BUILD index eb08dfcc3..8f0304a65 100644 --- a/common/BUILD +++ b/common/BUILD @@ -887,6 +887,7 @@ cc_test( "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", + "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", "@com_google_protobuf//:type_cc_proto", diff --git a/common/values/parsed_message_value.cc b/common/values/parsed_message_value.cc index 8a2b8030d..b0e881998 100644 --- a/common/values/parsed_message_value.cc +++ b/common/values/parsed_message_value.cc @@ -34,6 +34,7 @@ #include "base/attribute.h" #include "common/memory.h" #include "common/value.h" +#include "common/values/values.h" #include "extensions/protobuf/internal/qualify.h" #include "internal/empty_descriptors.h" #include "internal/json.h" @@ -51,8 +52,6 @@ namespace cel { namespace { -using ::cel::well_known_types::ValueReflection; - template std::enable_if_t, const google::protobuf::Message* absl_nonnull> @@ -114,12 +113,8 @@ absl::Status ParsedMessageValue::ConvertToJson( ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); - ValueReflection value_reflection; - CEL_RETURN_IF_ERROR(value_reflection.Initialize(json->GetDescriptor())); - google::protobuf::Message* json_object = value_reflection.MutableStructValue(json); - return internal::MessageToJson(*value_, descriptor_pool, message_factory, - json_object); + json); } absl::Status ParsedMessageValue::ConvertToJsonObject( diff --git a/common/values/parsed_message_value_test.cc b/common/values/parsed_message_value_test.cc index 7a84f82ba..1a6c3f628 100644 --- a/common/values/parsed_message_value_test.cc +++ b/common/values/parsed_message_value_test.cc @@ -14,6 +14,7 @@ #include +#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "absl/status/status_matchers.h" #include "absl/strings/cord.h" @@ -84,6 +85,19 @@ TEST_F(ParsedMessageValueTest, ConvertToJson) { R"pb(struct_value: {})pb")); } +TEST_F(ParsedMessageValueTest, ConvertToJsonFieldMask) { + ParsedMessageValue value = + MakeParsedMessage(R"pb(paths: "foo.bar" + paths: "baz")pb"); + google::protobuf::Message* json = + DynamicParseTextProto(R"pb()pb"); + ASSERT_THAT(value.ConvertToJson(descriptor_pool(), message_factory(), + cel::to_address(json)), + IsOk()); + EXPECT_THAT(*json, EqualsTextProto( + R"pb(string_value: "foo.bar,baz")pb")); +} + TEST_F(ParsedMessageValueTest, Equal) { MessageValue value = MakeParsedMessage(); EXPECT_THAT( diff --git a/conformance/BUILD b/conformance/BUILD index de1010628..a38e59c15 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -175,8 +175,6 @@ _TESTS_TO_SKIP = [ "namespace/namespace/self_eval_container_lookup,self_eval_container_lookup_unchecked", # Skip until fixed. - "wrappers/field_mask/to_json", - "wrappers/empty/to_json", "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", # Future features for CEL 1.0 diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 3ffe5994c..75eed88f3 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -71,6 +71,7 @@ cc_library( "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -111,9 +112,11 @@ cc_test( "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_absl//absl/types:span", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", + "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", "@com_google_protobuf//:wrappers_cc_proto", @@ -218,6 +221,7 @@ cc_test( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_absl//absl/types:span", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:empty_cc_proto", diff --git a/eval/public/structs/cel_proto_wrap_util.cc b/eval/public/structs/cel_proto_wrap_util.cc index 7bfe81fe6..490686fcb 100644 --- a/eval/public/structs/cel_proto_wrap_util.cc +++ b/eval/public/structs/cel_proto_wrap_util.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include "absl/base/optimization.h" #include "absl/functional/overload.h" #include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" @@ -50,6 +52,7 @@ #include "internal/well_known_types.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/json/json.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -79,6 +82,8 @@ using google::protobuf::Descriptor; using google::protobuf::DescriptorPool; using google::protobuf::Message; using google::protobuf::MessageFactory; +using google::protobuf::json::MessageToJsonString; +using google::protobuf::json::PrintOptions; // kMaxIntJSON is defined as the Number.MAX_SAFE_INTEGER value per EcmaScript 6. constexpr int64_t kMaxIntJSON = (1ll << 53) - 1; @@ -98,6 +103,35 @@ static bool IsJSONSafe(uint64_t i) { return i <= static_cast(kMaxIntJSON); } +static bool IsEmptyProto(const google::protobuf::Descriptor* descriptor) { + return descriptor->full_name() == "google.protobuf.Empty"; +} + +static bool IsFieldMaskProto(const google::protobuf::Descriptor* descriptor) { + return descriptor->full_name() == "google.protobuf.FieldMask"; +} + +static std::optional GetFieldMaskJsonString( + const google::protobuf::Message& message) { + // TODO(b/540507668): Refactor to pipe descriptor_pool through + // ValueFromValue to use internal::MessageToJson. + PrintOptions json_options; + std::string json_str; + auto status = MessageToJsonString(message, &json_str, json_options); + if (!status.ok()) { + ABSL_LOG(ERROR) << "Failed to convert FieldMask to JSON: " << status; + return std::nullopt; + } + // If JSON marshalling is correct, we know we'll always get a plain + // JSON string value and it shouldn't contain any escapes that we need + // to interpret. + if (json_str.size() >= 2 && json_str.front() == '"' && + json_str.back() == '"') { + return json_str.substr(1, json_str.size() - 2); + } + return json_str; +} + // Map implementation wrapping google.protobuf.ListValue class DynamicList : public CelList { public: @@ -1079,6 +1113,23 @@ google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, co return message; } } break; + case CelValue::Type::kMessage: { + const google::protobuf::Message* message_ptr = value.MessageOrDie(); + if (IsEmptyProto(message_ptr->GetDescriptor())) { + reflection.MutableStructValue(message); + return message; + } + if (IsFieldMaskProto(message_ptr->GetDescriptor())) { + std::optional fm_str = + GetFieldMaskJsonString(*message_ptr); + if (fm_str.has_value()) { + reflection.SetStringValue(message, *fm_str); + return message; + } + return nullptr; + } + return nullptr; + } break; case CelValue::Type::kNullType: reflection.SetNullValue(message); return message; @@ -1229,6 +1280,23 @@ bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* return ListFromValue(json->mutable_list_value(), value, arena); case CelValue::Type::kMap: return StructFromValue(json->mutable_struct_value(), value, arena); + case CelValue::Type::kMessage: { + const google::protobuf::Message* message_ptr = value.MessageOrDie(); + if (IsEmptyProto(message_ptr->GetDescriptor())) { + json->mutable_struct_value(); + return true; + } + if (IsFieldMaskProto(message_ptr->GetDescriptor())) { + std::optional fm_str = + GetFieldMaskJsonString(*message_ptr); + if (fm_str.has_value()) { + json->set_string_value(*fm_str); + return true; + } + return false; + } + return false; + } case CelValue::Type::kNullType: json->set_null_value(protobuf::NULL_VALUE); return true; @@ -1254,7 +1322,7 @@ google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototy case CelValue::Type::kBytes: { BytesValue v; type_name = v.GetTypeName(); - v.set_value(std::string(value.BytesOrDie().value())); + v.set_value(value.BytesOrDie().value()); payload = v.SerializeAsCord(); } break; case CelValue::Type::kDouble: { diff --git a/eval/public/structs/cel_proto_wrap_util_test.cc b/eval/public/structs/cel_proto_wrap_util_test.cc index 59597fe8f..b196d1dd3 100644 --- a/eval/public/structs/cel_proto_wrap_util_test.cc +++ b/eval/public/structs/cel_proto_wrap_util_test.cc @@ -15,6 +15,7 @@ #include "eval/public/structs/cel_proto_wrap_util.h" #include +#include #include #include #include @@ -24,12 +25,14 @@ #include "google/protobuf/any.pb.h" #include "google/protobuf/duration.pb.h" #include "google/protobuf/empty.pb.h" +#include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "google/protobuf/wrappers.pb.h" #include "absl/base/no_destructor.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" +#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" @@ -41,6 +44,7 @@ #include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" +#include "google/protobuf/arena.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" @@ -48,10 +52,12 @@ namespace google::api::expr::runtime::internal { namespace { +using ::google::protobuf::TextFormat; using ::testing::Eq; using ::testing::UnorderedPointwise; using google::protobuf::Duration; +using google::protobuf::FieldMask; using google::protobuf::ListValue; using google::protobuf::Struct; using google::protobuf::Timestamp; @@ -77,7 +83,7 @@ CelValue ProtobufValueFactoryImpl(const google::protobuf::Message* m) { class CelProtoWrapperTest : public ::testing::Test { protected: - CelProtoWrapperTest() {} + CelProtoWrapperTest() = default; void ExpectWrappedMessage(const CelValue& value, const google::protobuf::Message& message) { @@ -299,7 +305,7 @@ TEST_F(CelProtoWrapperTest, UnwrapMessageToValueStruct) { ASSERT_OK(missing_field_presence); EXPECT_FALSE(*missing_field_presence); - const CelList* key_list = cel_map->ListKeys().value(); + ASSERT_OK_AND_ASSIGN(const CelList* key_list, cel_map->ListKeys()); ASSERT_EQ(key_list->size(), kFields.size()); std::vector result_keys; @@ -436,6 +442,120 @@ TEST_F(CelProtoWrapperTest, UnwrapInvalidAny) { UnwrapMessageToValue(&any, &ProtobufValueFactoryImpl, arena()).IsError()); } +TEST_F(CelProtoWrapperTest, WrapFieldMaskToValue) { + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = ProtobufValueFactoryImpl(&field_mask); + + Value expected_message; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb(string_value: "foo.bar,baz")pb", + &expected_message)); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapMapWithFieldMaskToAny) { + const std::string kField = "field_mask"; + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = ProtobufValueFactoryImpl(&field_mask); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + CelValue cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + ASSERT_TRUE( + TextFormat::ParseFromString(R"pb( + fields { + key: "field_mask" + value { string_value: "foo.bar,baz" } + } + )pb", + &expected_struct)); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_struct)); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapListWithFieldMaskToAny) { + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = ProtobufValueFactoryImpl(&field_mask); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + ASSERT_TRUE(TextFormat::ParseFromString( + R"pb(values { string_value: "foo.bar,baz" })pb", &expected_list)); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_list)); + + ExpectWrappedMessage(list_value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapEmptyToValue) { + google::protobuf::Empty empty; + CelValue value = ProtobufValueFactoryImpl(&empty); + + Value expected_message; + expected_message.mutable_struct_value(); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapMapWithEmptyToAny) { + const std::string kField = "empty"; + google::protobuf::Empty empty; + CelValue value = ProtobufValueFactoryImpl(&empty); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + (*expected_struct.mutable_fields())[kField].mutable_struct_value(); + Any expected_message; + expected_message.PackFrom(expected_struct); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapListWithEmptyToAny) { + google::protobuf::Empty empty; + CelValue value = ProtobufValueFactoryImpl(&empty); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + expected_list.add_values()->mutable_struct_value(); + Any expected_message; + expected_message.PackFrom(expected_list); + + ExpectWrappedMessage(list_value, expected_message); +} // Test support of google.protobuf.Value wrappers in CelValue. TEST_F(CelProtoWrapperTest, UnwrapBoolWrapper) { bool value = true; @@ -797,10 +917,10 @@ TEST_F(CelProtoWrapperTest, WrapStruct) { std::vector> args = { {CelValue::CreateString(CelValue::StringHolder(&kField1)), CelValue::CreateBool(true)}}; - auto cel_map = + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + absl::Span>(args.data(), args.size()))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; @@ -817,10 +937,10 @@ TEST_F(CelProtoWrapperTest, WrapStruct) { TEST_F(CelProtoWrapperTest, WrapFailureStructBadKeyType) { std::vector> args = { {CelValue::CreateInt64(1L), CelValue::CreateBool(true)}}; - auto cel_map = + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + absl::Span>(args.data(), args.size()))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; @@ -833,10 +953,10 @@ TEST_F(CelProtoWrapperTest, WrapFailureStructBadValueType) { std::vector> args = { {CelValue::CreateString(CelValue::StringHolder(&kField1)), UnwrapMessageToValue(&bad_value, &ProtobufValueFactoryImpl, arena())}}; - auto cel_map = + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + absl::Span>(args.data(), args.size()))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; ExpectNotWrapped(cel_value, json); diff --git a/eval/public/structs/cel_proto_wrapper_test.cc b/eval/public/structs/cel_proto_wrapper_test.cc index b9fcd6b51..408e33284 100644 --- a/eval/public/structs/cel_proto_wrapper_test.cc +++ b/eval/public/structs/cel_proto_wrapper_test.cc @@ -1,6 +1,7 @@ #include "eval/public/structs/cel_proto_wrapper.h" #include +#include #include #include #include @@ -16,6 +17,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" +#include "absl/types/span.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" @@ -24,6 +26,7 @@ #include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" +#include "google/protobuf/arena.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/message.h" @@ -55,7 +58,7 @@ using google::protobuf::Arena; class CelProtoWrapperTest : public ::testing::Test { protected: - CelProtoWrapperTest() {} + CelProtoWrapperTest() = default; void ExpectWrappedMessage(const CelValue& value, const google::protobuf::Message& message) { @@ -290,7 +293,7 @@ TEST_F(CelProtoWrapperTest, UnwrapValueStruct) { ASSERT_OK(missing_field_presence); EXPECT_FALSE(*missing_field_presence); - const CelList* key_list = cel_map->ListKeys().value(); + ASSERT_OK_AND_ASSIGN(const CelList* key_list, cel_map->ListKeys()); ASSERT_EQ(key_list->size(), kFields.size()); std::vector result_keys; @@ -784,10 +787,10 @@ TEST_F(CelProtoWrapperTest, WrapStruct) { std::vector> args = { {CelValue::CreateString(CelValue::StringHolder(&kField1)), CelValue::CreateBool(true)}}; - auto cel_map = + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + absl::Span>(args.data(), args.size()))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; @@ -801,13 +804,60 @@ TEST_F(CelProtoWrapperTest, WrapStruct) { ExpectWrappedMessage(cel_value, any); } +TEST_F(CelProtoWrapperTest, WrapEmptyToValue) { + google::protobuf::Empty empty; + CelValue value = CelProtoWrapper::CreateMessage(&empty, arena()); + + Value expected_message; + expected_message.mutable_struct_value(); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapMapWithEmptyToAny) { + const std::string kField = "empty"; + google::protobuf::Empty empty; + CelValue value = CelProtoWrapper::CreateMessage(&empty, arena()); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + (*expected_struct.mutable_fields())[kField].mutable_struct_value(); + Any expected_message; + expected_message.PackFrom(expected_struct); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapperTest, WrapListWithEmptyToAny) { + google::protobuf::Empty empty; + CelValue value = CelProtoWrapper::CreateMessage(&empty, arena()); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + expected_list.add_values()->mutable_struct_value(); + Any expected_message; + expected_message.PackFrom(expected_list); + + ExpectWrappedMessage(list_value, expected_message); +} + TEST_F(CelProtoWrapperTest, WrapFailureStructBadKeyType) { std::vector> args = { {CelValue::CreateInt64(1L), CelValue::CreateBool(true)}}; - auto cel_map = + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + absl::Span>(args.data(), args.size()))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; @@ -820,10 +870,10 @@ TEST_F(CelProtoWrapperTest, WrapFailureStructBadValueType) { std::vector> args = { {CelValue::CreateString(CelValue::StringHolder(&kField1)), CelProtoWrapper::CreateMessage(&bad_value, arena())}}; - auto cel_map = + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, CreateContainerBackedMap( - absl::Span>(args.data(), args.size())) - .value(); + absl::Span>(args.data(), args.size()))); auto cel_value = CelValue::CreateMap(cel_map.get()); Value json; ExpectNotWrapped(cel_value, json); From fe0454f366b6081dde5d769936ae6346a4dd8fbd Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 10 Aug 2026 10:16:00 -0700 Subject: [PATCH 169/350] Update LegacyRuntimeTypeProvider to use modern ValueBuilder. PiperOrigin-RevId: 962225090 --- runtime/internal/BUILD | 6 -- .../internal/legacy_runtime_type_provider.cc | 95 +++++-------------- .../internal/legacy_runtime_type_provider.h | 8 +- .../legacy_runtime_type_provider_test.cc | 25 ++++- 4 files changed, 56 insertions(+), 78 deletions(-) diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index 44671102a..7247630bf 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -185,20 +185,14 @@ cc_library( hdrs = ["legacy_runtime_type_provider.h"], deps = [ "//common:legacy_value", - "//common:memory", "//common:type", "//common:value", "//eval/public:message_wrapper", - "//eval/public/structs:legacy_type_adapter", "//eval/public/structs:legacy_type_info_apis", - "//eval/public/structs:proto_message_type_adapter", "//eval/public/structs:protobuf_descriptor_type_provider", - "//extensions/protobuf:memory_manager", "//internal:status_macros", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", diff --git a/runtime/internal/legacy_runtime_type_provider.cc b/runtime/internal/legacy_runtime_type_provider.cc index 5db0b448e..f6f3c26ca 100644 --- a/runtime/internal/legacy_runtime_type_provider.cc +++ b/runtime/internal/legacy_runtime_type_provider.cc @@ -20,21 +20,15 @@ #include #include "absl/base/nullability.h" -#include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/legacy_value.h" -#include "common/memory.h" #include "common/type.h" #include "common/type_introspector.h" #include "common/value.h" +#include "common/values/value_builder.h" #include "eval/public/message_wrapper.h" -#include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" -#include "eval/public/structs/proto_message_type_adapter.h" -#include "extensions/protobuf/memory_manager.h" #include "internal/status_macros.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -44,61 +38,43 @@ namespace cel::runtime_internal { namespace { -using google::api::expr::runtime::LegacyTypeAdapter; using google::api::expr::runtime::LegacyTypeInfoApis; using google::api::expr::runtime::MessageWrapper; class LegacyValueBuilder final : public cel::ValueBuilder { public: - LegacyValueBuilder(cel::MemoryManagerRef memory_manager, - LegacyTypeAdapter adapter, MessageWrapper::Builder builder) - : memory_manager_(memory_manager), - adapter_(adapter), - builder_(std::move(builder)) {} + LegacyValueBuilder(google::protobuf::Arena* absl_nonnull arena, + cel::ValueBuilderPtr builder) + : arena_(arena), builder_(std::move(builder)) {} - absl::StatusOr> SetFieldByName( + absl::StatusOr> SetFieldByName( absl::string_view name, cel::Value value) override { - CEL_ASSIGN_OR_RETURN( - auto legacy_value, - LegacyValue(cel::extensions::ProtoMemoryManagerArena(memory_manager_), - value), - _.With(cel::ErrorValueReturn())); - CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetField( - name, legacy_value, memory_manager_, builder_)) - .With(cel::ErrorValueReturn()); - return std::nullopt; + return builder_->SetFieldByName(name, std::move(value)); } - absl::StatusOr> SetFieldByNumber( + absl::StatusOr> SetFieldByNumber( int64_t number, cel::Value value) override { - CEL_ASSIGN_OR_RETURN( - auto legacy_value, - LegacyValue(cel::extensions::ProtoMemoryManagerArena(memory_manager_), - value), - _.With(cel::ErrorValueReturn())); - CEL_RETURN_IF_ERROR(adapter_.mutation_apis()->SetFieldByNumber( - number, legacy_value, memory_manager_, builder_)) - .With(cel::ErrorValueReturn()); - return std::nullopt; + return builder_->SetFieldByNumber(number, std::move(value)); } absl::StatusOr Build() && override { - CEL_ASSIGN_OR_RETURN(auto value, - adapter_.mutation_apis()->AdaptFromWellKnownType( - memory_manager_, std::move(builder_)), + CEL_ASSIGN_OR_RETURN(auto value, std::move(*builder_).Build(), _.With(cel::ErrorValueReturn())); - CEL_ASSIGN_OR_RETURN( - auto result, - cel::ModernValue( - cel::extensions::ProtoMemoryManagerArena(memory_manager_), value), - _.With(cel::ErrorValueReturn())); - return result; + if (value.Is()) { + // Make the value behave like a legacy message. Minimizes further + // legacy/modern conversions (e.g. on return and when accessing fields). + CEL_ASSIGN_OR_RETURN(auto legacy_value, LegacyValue(arena_, value), + _.With(cel::ErrorValueReturn())); + CEL_ASSIGN_OR_RETURN(auto result, ModernValue(arena_, legacy_value), + _.With(cel::ErrorValueReturn())); + return result; + } + return value; } private: - cel::MemoryManagerRef memory_manager_; - LegacyTypeAdapter adapter_; - MessageWrapper::Builder builder_; + google::protobuf::Arena* const arena_; + cel::ValueBuilderPtr builder_; }; } // namespace @@ -108,26 +84,12 @@ LegacyRuntimeTypeProvider::NewValueBuilder( absl::string_view name, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { - auto type_adapter = ProvideLegacyType(name); - - if (!type_adapter.has_value()) { + auto builder = common_internal::NewValueBuilder(arena, descriptor_pool_, + message_factory, name); + if (builder == nullptr) { return nullptr; } - - // We know the implementation should not do this, but can't prove it to type - // system. - // Defensive checks but impractical to exercise. - const auto* mutation_apis = type_adapter->mutation_apis(); - if (mutation_apis == nullptr) { - return absl::FailedPreconditionError( - absl::StrCat("LegacyTypeMutationApis missing for type: ", name)); - } - - CEL_ASSIGN_OR_RETURN( - auto builder, - mutation_apis->NewInstance(cel::MemoryManagerRef::Pooling(arena))); - return std::make_unique( - cel::MemoryManagerRef::Pooling(arena), *type_adapter, std::move(builder)); + return std::make_unique(arena, std::move(builder)); } absl::StatusOr> LegacyRuntimeTypeProvider::FindTypeImpl( @@ -175,12 +137,7 @@ LegacyRuntimeTypeProvider::FindStructTypeFieldByNameImpl( field_desc->name, field_desc->number, cel::DynType{}); } - const auto* mutation_apis = (*type_info)->GetMutationApis(MessageWrapper()); - if (mutation_apis == nullptr || !mutation_apis->DefinesField(name)) { - return std::nullopt; - } - - return cel::common_internal::BasicStructTypeField(name, 0, cel::DynType{}); + return std::nullopt; } } // namespace cel::runtime_internal diff --git a/runtime/internal/legacy_runtime_type_provider.h b/runtime/internal/legacy_runtime_type_provider.h index 45aef5d75..ca277f3ec 100644 --- a/runtime/internal/legacy_runtime_type_provider.h +++ b/runtime/internal/legacy_runtime_type_provider.h @@ -37,8 +37,9 @@ class LegacyRuntimeTypeProvider final LegacyRuntimeTypeProvider( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory) - : google::api::expr::runtime::ProtobufDescriptorProvider( - descriptor_pool, message_factory) {} + : google::api::expr::runtime::ProtobufDescriptorProvider(descriptor_pool, + message_factory), + descriptor_pool_(descriptor_pool) {} absl::StatusOr NewValueBuilder( absl::string_view name, @@ -51,6 +52,9 @@ class LegacyRuntimeTypeProvider final absl::StatusOr> FindStructTypeFieldByNameImpl( absl::string_view type, absl::string_view name) const override; + + private: + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; }; } // namespace cel::runtime_internal diff --git a/runtime/internal/legacy_runtime_type_provider_test.cc b/runtime/internal/legacy_runtime_type_provider_test.cc index eceac65a9..a83f11390 100644 --- a/runtime/internal/legacy_runtime_type_provider_test.cc +++ b/runtime/internal/legacy_runtime_type_provider_test.cc @@ -86,7 +86,7 @@ TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByNameNotFound) { EXPECT_FALSE(field2.has_value()); } -TEST(LegacyRuntimeTypeProviderTest, NewValueBuilder) { +TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderMessage) { LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory()); google::protobuf::Arena arena; @@ -100,10 +100,33 @@ TEST(LegacyRuntimeTypeProviderTest, NewValueBuilder) { builder->SetFieldByName("single_int64", IntValue(42))); EXPECT_FALSE(field_result.has_value()); + ASSERT_OK_AND_ASSIGN(auto field_result2, + builder->SetFieldByNumber(1, IntValue(100))); + EXPECT_FALSE(field_result2.has_value()); + ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); EXPECT_TRUE(value.Is()); } +TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderWellKnownType) { + LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory()); + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(auto builder, + provider.NewValueBuilder( + "google.protobuf.Int64Value", + cel::internal::GetTestingMessageFactory(), &arena)); + ASSERT_NE(builder, nullptr); + + ASSERT_OK_AND_ASSIGN(auto field_result, + builder->SetFieldByName("value", IntValue(42))); + EXPECT_FALSE(field_result.has_value()); + + ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); + ASSERT_TRUE(value.Is()); + EXPECT_EQ(value.As()->NativeValue(), 42); +} + TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderNotFound) { LegacyRuntimeTypeProvider provider( google::protobuf::DescriptorPool::generated_pool(), From ad1eb7a16360d71d59b5dbb6ca8524eaa14b1111 Mon Sep 17 00:00:00 2001 From: Muhammad Askri Date: Mon, 10 Aug 2026 16:20:59 -0700 Subject: [PATCH 170/350] Enabling conformance testing for namespace shadowing PiperOrigin-RevId: 962428115 --- conformance/BUILD | 20 ++++++++++++++------ policy/yaml_policy_parser.cc | 3 +-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/conformance/BUILD b/conformance/BUILD index a38e59c15..a5889f83b 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -170,9 +170,9 @@ _TESTS_TO_SKIP = [ # Broken test cases which should be supported. # TODO(issues/97): Parse-only qualified variable lookup "x.y" with binding "x.y" or "y" within container "x" fails - "fields/qualified_identifier_resolution/qualified_ident,map_field_select,ident_with_longest_prefix_check,qualified_identifier_resolution_unchecked", - "namespace/qualified/self_eval_qualified_lookup", - "namespace/namespace/self_eval_container_lookup,self_eval_container_lookup_unchecked", + # NOTE: These tests have `disable_check: true` in their definition, making them force parse-only behavior even in checked mode test runs. + "fields/qualified_identifier_resolution/qualified_identifier_resolution_unchecked", + "namespace/namespace/self_eval_container_lookup_unchecked", # Skip until fixed. "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", @@ -190,10 +190,18 @@ _TESTS_TO_SKIP = [ # precision to preserve value. Not available on older compilers where we just use absl::Format. # We should probably update the spec to allow different formats that parse to the same value. "conversions/string/double_hard", +] - # Recent changes +_TESTS_TO_SKIP_PARSE_ONLY = [ + # TODO(issues/97): Parse-only qualified variable lookup "x.y" with binding "x.y" or "y" within container "x" fails. "namespace/namespace_shadowing/basic", "namespace/namespace_shadowing/comprehension_shadowing_namespaced_selector_disambiguation", + "fields/qualified_identifier_resolution/qualified_ident", + "fields/qualified_identifier_resolution/map_field_select", + "fields/qualified_identifier_resolution/ident_with_longest_prefix_check", + "namespace/qualified/self_eval_qualified_lookup", + "namespace/namespace/self_eval_container_lookup", + "type_deductions", ] _TESTS_TO_SKIP_MODERN = _TESTS_TO_SKIP @@ -238,14 +246,14 @@ gen_conformance_tests( name = "conformance_parse_only", data = _ALL_TESTS, modern = True, - skip_tests = _TESTS_TO_SKIP_MODERN + ["type_deductions"], + skip_tests = _TESTS_TO_SKIP_MODERN + _TESTS_TO_SKIP_PARSE_ONLY, ) gen_conformance_tests( name = "conformance_legacy_parse_only", data = _ALL_TESTS, modern = False, - skip_tests = _TESTS_TO_SKIP_LEGACY + ["type_deductions"], + skip_tests = _TESTS_TO_SKIP_LEGACY + _TESTS_TO_SKIP_PARSE_ONLY, ) gen_conformance_tests( diff --git a/policy/yaml_policy_parser.cc b/policy/yaml_policy_parser.cc index 37dfa1dad..d40cbd8d6 100644 --- a/policy/yaml_policy_parser.cc +++ b/policy/yaml_policy_parser.cc @@ -102,8 +102,7 @@ absl::Status YamlPolicyParser::ParsePolicy(CelPolicyParseContext& ctx) const { return absl::OkStatus(); } - // TODO(b/542282964): Fold this mapping into cel::Source decoding happens - // once. + // TODO(b/542282964): Fold this mapping into cel::Source decoding. std::string text = source->content().ToString(); std::vector mapping; mapping.resize(text.size() + 1, 0); From c680ec41bd72da1b45827c72286fd980a6c87d35 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 11 Aug 2026 10:31:15 -0700 Subject: [PATCH 171/350] Remove LegacyMutationApis This is now unused internally and no known references in other projects. PiperOrigin-RevId: 962861505 --- eval/eval/create_struct_step_test.cc | 3 +- eval/public/cel_type_registry_test.cc | 2 +- eval/public/structs/BUILD | 4 - eval/public/structs/legacy_type_adapter.h | 68 +---- .../structs/legacy_type_adapter_test.cc | 3 +- eval/public/structs/legacy_type_info_apis.h | 16 +- .../structs/legacy_type_provider_test.cc | 2 +- .../structs/proto_message_type_adapter.cc | 178 ------------ .../structs/proto_message_type_adapter.h | 42 +-- .../proto_message_type_adapter_test.cc | 265 +----------------- .../protobuf_descriptor_type_provider.cc | 3 +- .../protobuf_descriptor_type_provider_test.cc | 28 +- .../structs/trivial_legacy_type_info_test.cc | 7 - 13 files changed, 22 insertions(+), 599 deletions(-) diff --git a/eval/eval/create_struct_step_test.cc b/eval/eval/create_struct_step_test.cc index cd9db9bd9..666dddcdb 100644 --- a/eval/eval/create_struct_step_test.cc +++ b/eval/eval/create_struct_step_test.cc @@ -58,7 +58,6 @@ namespace { using ::absl_testing::IsOk; using ::absl_testing::StatusIs; -using ::cel::Expr; using ::cel::TypeProvider; using ::cel::internal::test::EqualsProto; using ::cel::runtime_internal::NewTestingRuntimeEnv; @@ -200,7 +199,7 @@ TEST_P(CreateCreateStructStepTest, TestEmptyMessageCreation) { auto adapter = env_->legacy_type_registry.FindTypeAdapter( "google.api.expr.runtime.TestMessage"); - ASSERT_TRUE(adapter.has_value() && adapter->mutation_apis() != nullptr); + ASSERT_TRUE(adapter.has_value() && adapter->access_apis() != nullptr); ASSERT_OK_AND_ASSIGN(auto maybe_type, env_->type_registry.GetComposedTypeProvider().FindType( diff --git a/eval/public/cel_type_registry_test.cc b/eval/public/cel_type_registry_test.cc index 881700d55..9dc8d946e 100644 --- a/eval/public/cel_type_registry_test.cc +++ b/eval/public/cel_type_registry_test.cc @@ -36,7 +36,7 @@ class TestTypeProvider : public LegacyTypeProvider { absl::string_view name) const override { for (const auto& type : types_) { if (name == type) { - return LegacyTypeAdapter(/*access=*/nullptr, /*mutation=*/nullptr); + return LegacyTypeAdapter(/*access=*/nullptr); } } return std::nullopt; diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 75eed88f3..14bd607a3 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -273,7 +273,6 @@ cc_library( srcs = ["proto_message_type_adapter.cc"], hdrs = ["proto_message_type_adapter.h"], deps = [ - ":cel_proto_wrap_util", ":field_access_impl", ":legacy_type_adapter", ":legacy_type_info_apis", @@ -311,12 +310,9 @@ cc_test( "//common:value_testing", "//eval/public:cel_value", "//eval/public:message_wrapper", - "//eval/public/containers:container_backed_list_impl", - "//eval/public/containers:container_backed_map_impl", "//eval/public/testing:matchers", "//eval/testutil:test_message_cc_proto", "//extensions/protobuf:memory_manager", - "//internal:proto_matchers", "//internal:testing", "//runtime:runtime_options", "@com_google_absl//absl/status", diff --git a/eval/public/structs/legacy_type_adapter.h b/eval/public/structs/legacy_type_adapter.h index 05fbc9a92..f526460d7 100644 --- a/eval/public/structs/legacy_type_adapter.h +++ b/eval/public/structs/legacy_type_adapter.h @@ -18,7 +18,7 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_LEGACY_TYPE_ADPATER_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_LEGACY_TYPE_ADPATER_H_ -#include +#include #include #include "absl/status/status.h" @@ -36,54 +36,6 @@ namespace google::api::expr::runtime { class DucktypedMessageAdapter; class ProtoMessageTypeAdapter; -// Interface for mutation apis. -// Note: in the new type system, a type provider represents this by returning -// a cel::Type and cel::ValueManager for the type. -class LegacyTypeMutationApis { - public: - virtual ~LegacyTypeMutationApis() = default; - - // Return whether the type defines the given field. - // TODO(uncreated-issue/3): This is only used to eagerly fail during the planning - // phase. Check if it's safe to remove this behavior and fail at runtime. - virtual bool DefinesField(absl::string_view field_name) const = 0; - - // Create a new empty instance of the type. - // May return a status if the type is not possible to create. - virtual absl::StatusOr NewInstance( - cel::MemoryManagerRef memory_manager) const = 0; - - // Normalize special types to a native CEL value after building. - // The interpreter guarantees that instance is uniquely owned by the - // interpreter, and can be safely mutated. - virtual absl::StatusOr AdaptFromWellKnownType( - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder instance) const = 0; - - // Set field on instance to value. - // The interpreter guarantees that instance is uniquely owned by the - // interpreter, and can be safely mutated. - virtual absl::Status SetField( - absl::string_view field_name, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const = 0; - - virtual absl::Status SetFieldByNumber( - int64_t field_number [[maybe_unused]], - const CelValue& value [[maybe_unused]], - cel::MemoryManagerRef memory_manager [[maybe_unused]], - CelValue::MessageWrapper::Builder& instance [[maybe_unused]]) const { - return absl::UnimplementedError("SetFieldByNumber is not yet implemented"); - } - - private: - // This class should only be implemented by CEL. Custom structs are only - // supported using the cel::Value APIs. - friend class ProtoMessageTypeAdapter; - - LegacyTypeMutationApis() = default; -}; - // Interface for access apis. // Note: in new type system this is integrated into the StructValue (via // dynamic dispatch to concrete implementations). @@ -162,9 +114,6 @@ class LegacyTypeAccessApis { // Type information about a legacy Struct type. // Provides methods to the interpreter for interacting with a custom type. // -// mutation_apis() provide equivalent behavior to a cel::Type and -// cel::ValueManager (resolved from a type name). -// // access_apis() provide equivalent behavior to cel::StructValue accessors // (virtual dispatch to a concrete implementation for accessing underlying // values). @@ -174,21 +123,18 @@ class LegacyTypeAccessApis { // the type provider that returned this object. class LegacyTypeAdapter { public: - LegacyTypeAdapter(const LegacyTypeAccessApis* access, - const LegacyTypeMutationApis* mutation) - : access_apis_(access), mutation_apis_(mutation) {} + explicit LegacyTypeAdapter(const LegacyTypeAccessApis* access) + : access_apis_(access) {} + // Temporary constructor to support fakes in client tests. + LegacyTypeAdapter(const LegacyTypeAccessApis* access, std::nullptr_t) + : access_apis_(access) {} // Apis for access for the represented type. // If null, access is not supported (this is an opaque type). - const LegacyTypeAccessApis* access_apis() { return access_apis_; } - - // Apis for mutation for the represented type. - // If null, mutation is not supported (this type cannot be created). - const LegacyTypeMutationApis* mutation_apis() { return mutation_apis_; } + const LegacyTypeAccessApis* access_apis() const { return access_apis_; } private: const LegacyTypeAccessApis* access_apis_; - const LegacyTypeMutationApis* mutation_apis_; }; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/legacy_type_adapter_test.cc b/eval/public/structs/legacy_type_adapter_test.cc index 1dabac7d7..603087b6b 100644 --- a/eval/public/structs/legacy_type_adapter_test.cc +++ b/eval/public/structs/legacy_type_adapter_test.cc @@ -23,10 +23,9 @@ namespace { TEST(LegacyTypeAdapter, Basic) { ProtoMessageTypeAdapter adapter(TestMessage::descriptor(), nullptr); - LegacyTypeAdapter type_adapter(&adapter, &adapter); + LegacyTypeAdapter type_adapter(&adapter); EXPECT_EQ(type_adapter.access_apis(), &adapter); - EXPECT_EQ(type_adapter.mutation_apis(), &adapter); } } // namespace diff --git a/eval/public/structs/legacy_type_info_apis.h b/eval/public/structs/legacy_type_info_apis.h index 90e2f3710..d7f5db3fa 100644 --- a/eval/public/structs/legacy_type_info_apis.h +++ b/eval/public/structs/legacy_type_info_apis.h @@ -27,7 +27,6 @@ namespace google::api::expr::runtime { // Forward declared to resolve cyclic dependency. class LegacyTypeAccessApis; -class LegacyTypeMutationApis; // Forward declare permitted subclasses. class DucktypedMessageAdapter; @@ -40,8 +39,8 @@ class TrivialTypeInfo; // Provides ability to obtain field access apis, type info, and debug // representation of a message. // -// The message parameter may wrap a nullptr to request generic accessors / -// mutators for the TypeInfo instance if it is available. +// The message parameter may wrap a nullptr to request generic accessors for +// the TypeInfo instance if it is available. // // This is implemented as a separate class from LegacyTypeAccessApis to resolve // cyclic dependency between CelValue (which needs to access these apis to @@ -87,17 +86,6 @@ class LegacyTypeInfoApis { virtual const LegacyTypeAccessApis* GetAccessApis( const MessageWrapper& wrapped_message) const = 0; - // Return a pointer to the wrapped message's mutation api implementation. - // - // The CEL interpreter assumes that the returned pointer is owned externally - // and will outlive any CelValues created by the interpreter. - // - // Nullptr signals that the value does not provide mutation apis. - virtual const LegacyTypeMutationApis* GetMutationApis( - const MessageWrapper& wrapped_message [[maybe_unused]]) const { - return nullptr; - } - // Return a description of the underlying field if defined. // // The underlying string is expected to remain valid as long as the diff --git a/eval/public/structs/legacy_type_provider_test.cc b/eval/public/structs/legacy_type_provider_test.cc index c8059da7c..59246fed5 100644 --- a/eval/public/structs/legacy_type_provider_test.cc +++ b/eval/public/structs/legacy_type_provider_test.cc @@ -41,7 +41,7 @@ class LegacyTypeProviderTestImpl : public LegacyTypeProvider { absl::optional ProvideLegacyType( absl::string_view name) const override { if (name == "test") { - return LegacyTypeAdapter(nullptr, nullptr); + return LegacyTypeAdapter(nullptr); } return std::nullopt; } diff --git a/eval/public/structs/proto_message_type_adapter.cc b/eval/public/structs/proto_message_type_adapter.cc index db5f3c6e2..2f72ec68a 100644 --- a/eval/public/structs/proto_message_type_adapter.cc +++ b/eval/public/structs/proto_message_type_adapter.cc @@ -26,7 +26,6 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" #include "absl/types/span.h" #include "base/attribute.h" #include "common/memory.h" @@ -35,7 +34,6 @@ #include "eval/public/containers/internal_field_backed_list_impl.h" #include "eval/public/containers/internal_field_backed_map_impl.h" #include "eval/public/message_wrapper.h" -#include "eval/public/structs/cel_proto_wrap_util.h" #include "eval/public/structs/field_access_impl.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" @@ -360,11 +358,6 @@ class DucktypedMessageAdapter : public LegacyTypeAccessApis, return this; } - const LegacyTypeMutationApis* GetMutationApis( - const MessageWrapper& wrapped_message) const override { - return nullptr; - } - static const DucktypedMessageAdapter& GetSingleton() { static absl::NoDestructor instance; return *instance; @@ -418,12 +411,6 @@ absl::string_view ProtoMessageTypeAdapter::GetTypename( return descriptor_->full_name(); } -const LegacyTypeMutationApis* ProtoMessageTypeAdapter::GetMutationApis( - const MessageWrapper& wrapped_message) const { - // Defer checks for misuse on wrong message kind in the accessor calls. - return this; -} - const LegacyTypeAccessApis* ProtoMessageTypeAdapter::GetAccessApis( const MessageWrapper& wrapped_message) const { // Defer checks for misuse on wrong message kind in the builder calls. @@ -447,41 +434,6 @@ ProtoMessageTypeAdapter::FindFieldByName(absl::string_view field_name) const { field_descriptor->name()}; } -absl::Status ProtoMessageTypeAdapter::ValidateSetFieldOp( - bool assertion, absl::string_view field, absl::string_view detail) const { - if (!assertion) { - return absl::InvalidArgumentError( - absl::Substitute("SetField failed on message $0, field '$1': $2", - descriptor_->full_name(), field, detail)); - } - return absl::OkStatus(); -} - -absl::StatusOr -ProtoMessageTypeAdapter::NewInstance( - cel::MemoryManagerRef memory_manager) const { - if (message_factory_ == nullptr) { - return absl::UnimplementedError( - absl::StrCat("Cannot create message ", descriptor_->name())); - } - - // This implementation requires arena-backed memory manager. - google::protobuf::Arena* arena = ProtoMemoryManagerArena(memory_manager); - const Message* prototype = message_factory_->GetPrototype(descriptor_); - - Message* msg = (prototype != nullptr) ? prototype->New(arena) : nullptr; - - if (msg == nullptr) { - return absl::InvalidArgumentError( - absl::StrCat("Failed to create message ", descriptor_->name())); - } - return MessageWrapper::Builder(msg); -} - -bool ProtoMessageTypeAdapter::DefinesField(absl::string_view field_name) const { - return descriptor_->FindFieldByName(field_name) != nullptr; -} - absl::StatusOr ProtoMessageTypeAdapter::HasField( absl::string_view field_name, const CelValue::MessageWrapper& value) const { CEL_ASSIGN_OR_RETURN(const google::protobuf::Message* message, @@ -512,136 +464,6 @@ ProtoMessageTypeAdapter::Qualify( memory_manager); } -absl::Status ProtoMessageTypeAdapter::SetField( - const google::protobuf::FieldDescriptor* field, const CelValue& value, - google::protobuf::Arena* arena, google::protobuf::Message* message) const { - if (field->is_map()) { - constexpr int kKeyField = 1; - constexpr int kValueField = 2; - - const CelMap* cel_map; - CEL_RETURN_IF_ERROR(ValidateSetFieldOp( - value.GetValue(&cel_map) && cel_map != nullptr, - field->name(), - absl::StrCat("value is not CelMap - value is ", - CelValue::TypeName(value.type())))); - - auto entry_descriptor = field->message_type(); - - CEL_RETURN_IF_ERROR( - ValidateSetFieldOp(entry_descriptor != nullptr, field->name(), - "failed to find map entry descriptor")); - auto key_field_descriptor = entry_descriptor->FindFieldByNumber(kKeyField); - auto value_field_descriptor = - entry_descriptor->FindFieldByNumber(kValueField); - - CEL_RETURN_IF_ERROR( - ValidateSetFieldOp(key_field_descriptor != nullptr, field->name(), - "failed to find key field descriptor")); - - CEL_RETURN_IF_ERROR( - ValidateSetFieldOp(value_field_descriptor != nullptr, field->name(), - "failed to find value field descriptor")); - - bool prune_when_null = false; - if (value_field_descriptor->cpp_type() == - google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { - auto well_known_type = - value_field_descriptor->message_type()->well_known_type(); - if (well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_ANY && - well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE && - well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE && - well_known_type != google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT) { - prune_when_null = true; - } - } - - CEL_ASSIGN_OR_RETURN(const CelList* key_list, cel_map->ListKeys(arena)); - for (int i = 0; i < key_list->size(); i++) { - CelValue key = (*key_list).Get(arena, i); - - auto value = (*cel_map).Get(arena, key); - CEL_RETURN_IF_ERROR(ValidateSetFieldOp(value.has_value(), field->name(), - "error serializing CelMap")); - if (prune_when_null && value->IsNull()) { - continue; - } - Message* entry_msg = message->GetReflection()->AddMessage(message, field); - CEL_RETURN_IF_ERROR(internal::SetValueToSingleField( - key, key_field_descriptor, entry_msg, arena)); - CEL_RETURN_IF_ERROR(internal::SetValueToSingleField( - value.value(), value_field_descriptor, entry_msg, arena)); - } - - } else if (field->is_repeated()) { - const CelList* cel_list; - CEL_RETURN_IF_ERROR(ValidateSetFieldOp( - value.GetValue(&cel_list) && cel_list != nullptr, - field->name(), - absl::StrCat("expected CelList value - value is", - CelValue::TypeName(value.type())))); - - for (int i = 0; i < cel_list->size(); i++) { - CEL_RETURN_IF_ERROR(internal::AddValueToRepeatedField( - (*cel_list).Get(arena, i), field, message, arena)); - } - } else { - CEL_RETURN_IF_ERROR( - internal::SetValueToSingleField(value, field, message, arena)); - } - return absl::OkStatus(); -} - -absl::Status ProtoMessageTypeAdapter::SetField( - absl::string_view field_name, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const { - // Assume proto arena implementation if this provider is used. - google::protobuf::Arena* arena = - cel::extensions::ProtoMemoryManagerArena(memory_manager); - - CEL_ASSIGN_OR_RETURN(google::protobuf::Message * mutable_message, - UnwrapMessage(instance, "SetField")); - - const google::protobuf::FieldDescriptor* field_descriptor = - descriptor_->FindFieldByName(field_name); - CEL_RETURN_IF_ERROR( - ValidateSetFieldOp(field_descriptor != nullptr, field_name, "not found")); - - return SetField(field_descriptor, value, arena, mutable_message); -} - -absl::Status ProtoMessageTypeAdapter::SetFieldByNumber( - int64_t field_number, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const { - // Assume proto arena implementation if this provider is used. - google::protobuf::Arena* arena = - cel::extensions::ProtoMemoryManagerArena(memory_manager); - - CEL_ASSIGN_OR_RETURN(google::protobuf::Message * mutable_message, - UnwrapMessage(instance, "SetField")); - - const google::protobuf::FieldDescriptor* field_descriptor = - descriptor_->FindFieldByNumber(field_number); - CEL_RETURN_IF_ERROR(ValidateSetFieldOp( - field_descriptor != nullptr, absl::StrCat(field_number), "not found")); - - return SetField(field_descriptor, value, arena, mutable_message); -} - -absl::StatusOr ProtoMessageTypeAdapter::AdaptFromWellKnownType( - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder instance) const { - // Assume proto arena implementation if this provider is used. - google::protobuf::Arena* arena = - cel::extensions::ProtoMemoryManagerArena(memory_manager); - CEL_ASSIGN_OR_RETURN(google::protobuf::Message * message, - UnwrapMessage(instance, "AdaptFromWellKnownType")); - return internal::UnwrapMessageToValue(message, &MessageCelValueFactory, - arena); -} - bool ProtoMessageTypeAdapter::IsEqualTo( const CelValue::MessageWrapper& instance, const CelValue::MessageWrapper& other_instance) const { diff --git a/eval/public/structs/proto_message_type_adapter.h b/eval/public/structs/proto_message_type_adapter.h index f4e6b4c8b..867ba39cf 100644 --- a/eval/public/structs/proto_message_type_adapter.h +++ b/eval/public/structs/proto_message_type_adapter.h @@ -38,12 +38,12 @@ namespace google::api::expr::runtime { // generally the duck-typed instance to support the default behavior of // deferring to the protobuf reflection apis on the message instance. class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, - public LegacyTypeAccessApis, - public LegacyTypeMutationApis { + public LegacyTypeAccessApis { public: - ProtoMessageTypeAdapter(const google::protobuf::Descriptor* absl_nonnull descriptor, - google::protobuf::MessageFactory* message_factory) - : message_factory_(message_factory), descriptor_(descriptor) {} + explicit ProtoMessageTypeAdapter( + const google::protobuf::Descriptor* absl_nonnull descriptor, + google::protobuf::MessageFactory* message_factory = nullptr) + : descriptor_(descriptor) {} ~ProtoMessageTypeAdapter() override = default; @@ -61,32 +61,9 @@ class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, const LegacyTypeAccessApis* GetAccessApis( const MessageWrapper& wrapped_message) const override; - const LegacyTypeMutationApis* GetMutationApis( - const MessageWrapper& wrapped_message) const override; - absl::optional FindFieldByName( absl::string_view field_name) const override; - // Implement LegacyTypeMutation APIs. - absl::StatusOr NewInstance( - cel::MemoryManagerRef memory_manager) const override; - - bool DefinesField(absl::string_view field_name) const override; - - absl::Status SetField( - absl::string_view field_name, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const override; - - absl::Status SetFieldByNumber( - int64_t field_number, const CelValue& value, - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder& instance) const override; - - absl::StatusOr AdaptFromWellKnownType( - cel::MemoryManagerRef memory_manager, - CelValue::MessageWrapper::Builder instance) const override; - // Implement LegacyTypeAccessAPIs. absl::StatusOr GetField( absl::string_view field_name, const CelValue::MessageWrapper& instance, @@ -113,15 +90,6 @@ class ProtoMessageTypeAdapter : public LegacyTypeInfoApis, } private: - // Helper for standardizing error messages for SetField operation. - absl::Status ValidateSetFieldOp(bool assertion, absl::string_view field, - absl::string_view detail) const; - - absl::Status SetField(const google::protobuf::FieldDescriptor* field, - const CelValue& value, google::protobuf::Arena* arena, - google::protobuf::Message* message) const; - - google::protobuf::MessageFactory* message_factory_; const google::protobuf::Descriptor* absl_nonnull descriptor_; }; diff --git a/eval/public/structs/proto_message_type_adapter_test.cc b/eval/public/structs/proto_message_type_adapter_test.cc index c0e60c632..a74a32cb4 100644 --- a/eval/public/structs/proto_message_type_adapter_test.cc +++ b/eval/public/structs/proto_message_type_adapter_test.cc @@ -14,6 +14,7 @@ #include "eval/public/structs/proto_message_type_adapter.h" +#include #include #include "google/protobuf/wrappers.pb.h" @@ -24,15 +25,12 @@ #include "common/value.h" #include "common/value_testing.h" #include "eval/public/cel_value.h" -#include "eval/public/containers/container_backed_list_impl.h" -#include "eval/public/containers/container_backed_map_impl.h" #include "eval/public/message_wrapper.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" #include "eval/public/testing/matchers.h" #include "eval/testutil/test_message.pb.h" #include "extensions/protobuf/memory_manager.h" -#include "internal/proto_matchers.h" #include "internal/testing.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -48,12 +46,9 @@ using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::cel::ProtoWrapperTypeOptions; using ::cel::extensions::ProtoMemoryManagerRef; -using ::cel::internal::test::EqualsProto; using ::google::protobuf::Int64Value; using ::testing::_; -using ::testing::AllOf; using ::testing::ElementsAre; -using ::testing::Eq; using ::testing::Field; using ::testing::HasSubstr; using ::testing::Optional; @@ -427,247 +422,6 @@ TEST(GetGenericProtoTypeInfoInstance, FallbackForNonMessage) { EXPECT_EQ(info_api.DebugString(null_message), ""); } -TEST(ProtoMessageTypeAdapter, NewInstance) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder result, - adapter.NewInstance(manager)); - EXPECT_EQ(result.message_ptr()->SerializeAsString(), ""); -} - -TEST(ProtoMessageTypeAdapter, NewInstanceUnsupportedDescriptor) { - google::protobuf::Arena arena; - - google::protobuf::DescriptorPool pool; - google::protobuf::FileDescriptorProto faked_file; - faked_file.set_name("faked.proto"); - faked_file.set_syntax("proto3"); - faked_file.set_package("google.api.expr.runtime"); - auto msg_descriptor = faked_file.add_message_type(); - msg_descriptor->set_name("FakeMessage"); - pool.BuildFile(faked_file); - - ProtoMessageTypeAdapter adapter( - pool.FindMessageTypeByName("google.api.expr.runtime.FakeMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - // Message factory doesn't know how to create our custom message, even though - // we provided a descriptor for it. - EXPECT_THAT( - adapter.NewInstance(manager), - StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("FakeMessage"))); -} - -TEST(ProtoMessageTypeAdapter, DefinesField) { - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - - EXPECT_TRUE(adapter.DefinesField("int64_value")); - EXPECT_FALSE(adapter.DefinesField("not_a_field")); -} - -TEST(ProtoMessageTypeAdapter, SetFieldSingular) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder value, - adapter.NewInstance(manager)); - - ASSERT_OK(adapter.SetField("int64_value", CelValue::CreateInt64(10), manager, - value)); - - TestMessage message; - message.set_int64_value(10); - EXPECT_EQ(value.message_ptr()->SerializeAsString(), - message.SerializeAsString()); - - ASSERT_THAT(adapter.SetField("not_a_field", CelValue::CreateInt64(10), - manager, value), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("field 'not_a_field': not found"))); -} - -TEST(ProtoMessageTypeAdapter, SetFieldRepeated) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ContainerBackedListImpl list( - {CelValue::CreateInt64(1), CelValue::CreateInt64(2)}); - CelValue value_to_set = CelValue::CreateList(&list); - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - - ASSERT_OK(adapter.SetField("int64_list", value_to_set, manager, instance)); - - TestMessage message; - message.add_int64_list(1); - message.add_int64_list(2); - - EXPECT_EQ(instance.message_ptr()->SerializeAsString(), - message.SerializeAsString()); -} - -TEST(ProtoMessageTypeAdapter, SetFieldNotAField) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - - ASSERT_THAT(adapter.SetField("not_a_field", CelValue::CreateInt64(10), - manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("field 'not_a_field': not found"))); -} - -TEST(ProtoMesssageTypeAdapter, SetFieldWrongType) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ContainerBackedListImpl list( - {CelValue::CreateInt64(1), CelValue::CreateInt64(2)}); - CelValue list_value = CelValue::CreateList(&list); - - CelMapBuilder builder; - ASSERT_OK(builder.Add(CelValue::CreateInt64(1), CelValue::CreateInt64(2))); - ASSERT_OK(builder.Add(CelValue::CreateInt64(2), CelValue::CreateInt64(4))); - - CelValue map_value = CelValue::CreateMap(&builder); - - CelValue int_value = CelValue::CreateInt64(42); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - - EXPECT_THAT(adapter.SetField("int64_value", map_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_THAT(adapter.SetField("int64_value", list_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - - EXPECT_THAT( - adapter.SetField("int64_int32_map", list_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_THAT(adapter.SetField("int64_int32_map", int_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - - EXPECT_THAT(adapter.SetField("int64_list", int_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_THAT(adapter.SetField("int64_list", map_value, manager, instance), - StatusIs(absl::StatusCode::kInvalidArgument)); -} - -TEST(ProtoMesssageTypeAdapter, SetFieldNotAMessage) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - CelValue int_value = CelValue::CreateInt64(42); - CelValue::MessageWrapper::Builder instance( - static_cast(nullptr)); - - EXPECT_THAT(adapter.SetField("int64_value", int_value, manager, instance), - StatusIs(absl::StatusCode::kInternal)); -} - -TEST(ProtoMesssageTypeAdapter, SetFieldNullMessage) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - CelValue int_value = CelValue::CreateInt64(42); - CelValue::MessageWrapper::Builder instance( - static_cast(nullptr)); - - EXPECT_THAT(adapter.SetField("int64_value", int_value, manager, instance), - StatusIs(absl::StatusCode::kInternal)); -} - -TEST(ProtoMessageTypeAdapter, AdaptFromWellKnownType) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.protobuf.Int64Value"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - ASSERT_OK( - adapter.SetField("value", CelValue::CreateInt64(42), manager, instance)); - - ASSERT_OK_AND_ASSIGN(CelValue value, - adapter.AdaptFromWellKnownType(manager, instance)); - - EXPECT_THAT(value, test::IsCelInt64(42)); -} - -TEST(ProtoMessageTypeAdapter, AdaptFromWellKnownTypeUnspecial) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder instance, - adapter.NewInstance(manager)); - - ASSERT_OK(adapter.SetField("int64_value", CelValue::CreateInt64(42), manager, - instance)); - ASSERT_OK_AND_ASSIGN(CelValue value, - adapter.AdaptFromWellKnownType(manager, instance)); - - // TestMessage should not be converted to a CEL primitive type. - EXPECT_THAT(value, test::IsCelMessage(EqualsProto("int64_value: 42"))); -} - -TEST(ProtoMessageTypeAdapter, AdaptFromWellKnownTypeNotAMessageError) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - CelValue::MessageWrapper::Builder instance( - static_cast(nullptr)); - - // Interpreter guaranteed to call this with a message type, otherwise, - // something has broken. - EXPECT_THAT(adapter.AdaptFromWellKnownType(manager, instance), - StatusIs(absl::StatusCode::kInternal)); -} - TEST(ProtoMesssageTypeAdapter, TypeInfoDebug) { ProtoMessageTypeAdapter adapter( google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( @@ -716,23 +470,6 @@ TEST(ProtoMesssageTypeAdapter, FindFieldNotFound) { EXPECT_EQ(adapter.FindFieldByName("foo_not_a_field"), std::nullopt); } -TEST(ProtoMesssageTypeAdapter, TypeInfoMutator) { - google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); - auto manager = ProtoMemoryManagerRef(&arena); - - const LegacyTypeMutationApis* api = adapter.GetMutationApis(MessageWrapper()); - ASSERT_NE(api, nullptr); - - ASSERT_OK_AND_ASSIGN(MessageWrapper::Builder builder, - api->NewInstance(manager)); - EXPECT_NE(google::protobuf::DynamicCastMessage(builder.message_ptr()), - nullptr); -} - TEST(ProtoMesssageTypeAdapter, TypeInfoAccesor) { google::protobuf::Arena arena; ProtoMessageTypeAdapter adapter( diff --git a/eval/public/structs/protobuf_descriptor_type_provider.cc b/eval/public/structs/protobuf_descriptor_type_provider.cc index b5746523e..37fc8ce77 100644 --- a/eval/public/structs/protobuf_descriptor_type_provider.cc +++ b/eval/public/structs/protobuf_descriptor_type_provider.cc @@ -29,8 +29,7 @@ absl::optional ProtobufDescriptorProvider::ProvideLegacyType( if (result == nullptr) { return std::nullopt; } - // ProtoMessageTypeAdapter provides apis for both access and mutation. - return LegacyTypeAdapter(result, result); + return LegacyTypeAdapter(result); } absl::optional diff --git a/eval/public/structs/protobuf_descriptor_type_provider_test.cc b/eval/public/structs/protobuf_descriptor_type_provider_test.cc index f2b7900ef..ed1ddb811 100644 --- a/eval/public/structs/protobuf_descriptor_type_provider_test.cc +++ b/eval/public/structs/protobuf_descriptor_type_provider_test.cc @@ -17,34 +17,25 @@ #include #include "google/protobuf/wrappers.pb.h" -#include "absl/status/status_matchers.h" #include "eval/public/cel_value.h" #include "eval/public/structs/legacy_type_info_apis.h" -#include "eval/public/testing/matchers.h" -#include "extensions/protobuf/memory_manager.h" #include "internal/testing.h" -#include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace google::api::expr::runtime { namespace { -using ::absl_testing::IsOk; -using ::cel::extensions::ProtoMemoryManager; - TEST(ProtobufDescriptorProvider, Basic) { ProtobufDescriptorProvider provider( google::protobuf::DescriptorPool::generated_pool(), google::protobuf::MessageFactory::generated_factory()); - google::protobuf::Arena arena; - auto manager = ProtoMemoryManager(&arena); auto type_adapter = provider.ProvideLegacyType("google.protobuf.Int64Value"); std::optional type_info = provider.ProvideLegacyTypeInfo("google.protobuf.Int64Value"); ASSERT_TRUE(type_adapter.has_value()); - ASSERT_TRUE(type_adapter->mutation_apis() != nullptr); + ASSERT_TRUE(type_adapter->access_apis() != nullptr); ASSERT_TRUE(type_info.has_value()); ASSERT_TRUE(type_info != nullptr); @@ -52,20 +43,6 @@ TEST(ProtobufDescriptorProvider, Basic) { CelValue::MessageWrapper int64_cel_value(&int64_value, *type_info); EXPECT_EQ((*type_info)->GetTypename(int64_cel_value), "google.protobuf.Int64Value"); - - ASSERT_TRUE(type_adapter->mutation_apis()->DefinesField("value")); - ASSERT_OK_AND_ASSIGN(CelValue::MessageWrapper::Builder value, - type_adapter->mutation_apis()->NewInstance(manager)); - - ASSERT_THAT(type_adapter->mutation_apis()->SetField( - "value", CelValue::CreateInt64(10), manager, value), - IsOk()); - - ASSERT_OK_AND_ASSIGN( - CelValue adapted, - type_adapter->mutation_apis()->AdaptFromWellKnownType(manager, value)); - - EXPECT_THAT(adapted, test::IsCelInt64(10)); } // This is an implementation detail, but testing for coverage. @@ -76,12 +53,11 @@ TEST(ProtobufDescriptorProvider, MemoizesAdapters) { auto type_adapter = provider.ProvideLegacyType("google.protobuf.Int64Value"); ASSERT_TRUE(type_adapter.has_value()); - ASSERT_TRUE(type_adapter->mutation_apis() != nullptr); + ASSERT_TRUE(type_adapter->access_apis() != nullptr); auto type_adapter2 = provider.ProvideLegacyType("google.protobuf.Int64Value"); ASSERT_TRUE(type_adapter2.has_value()); - EXPECT_EQ(type_adapter->mutation_apis(), type_adapter2->mutation_apis()); EXPECT_EQ(type_adapter->access_apis(), type_adapter2->access_apis()); } diff --git a/eval/public/structs/trivial_legacy_type_info_test.cc b/eval/public/structs/trivial_legacy_type_info_test.cc index 9cc6e4916..ea7b3977e 100644 --- a/eval/public/structs/trivial_legacy_type_info_test.cc +++ b/eval/public/structs/trivial_legacy_type_info_test.cc @@ -44,13 +44,6 @@ TEST(TrivialTypeInfo, GetAccessApis) { EXPECT_EQ(TrivialTypeInfo::GetInstance()->GetAccessApis(wrapper), nullptr); } -TEST(TrivialTypeInfo, GetMutationApis) { - TrivialTypeInfo info; - MessageWrapper wrapper; - - EXPECT_EQ(info.GetMutationApis(wrapper), nullptr); - EXPECT_EQ(TrivialTypeInfo::GetInstance()->GetMutationApis(wrapper), nullptr); -} TEST(TrivialTypeInfo, FindFieldByName) { TrivialTypeInfo info; From 7341cacbc2b7c393d88d05704dddda4aaf77f8c8 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 11 Aug 2026 15:52:11 -0700 Subject: [PATCH 172/350] Update the LegacyTypeProvider to proxy to the normal implementation for field lookups. PiperOrigin-RevId: 963046621 --- eval/compiler/BUILD | 2 - eval/compiler/flat_expr_builder.cc | 2 +- eval/public/BUILD | 1 + eval/public/cel_type_registry.h | 21 ++++++- runtime/internal/BUILD | 8 +-- .../internal/legacy_runtime_type_provider.cc | 56 ------------------- .../internal/legacy_runtime_type_provider.h | 26 +++++---- .../legacy_runtime_type_provider_test.cc | 34 ++++++----- runtime/type_registry.cc | 4 +- runtime/type_registry.h | 18 +++--- 10 files changed, 66 insertions(+), 106 deletions(-) diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 2012abeda..dbd8d9c8b 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -139,7 +139,6 @@ cc_library( "//runtime/internal:issue_collector", "//runtime/internal:runtime_env", "@com_google_absl//absl/algorithm:container", - "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -150,7 +149,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_absl//absl/types:variant", "@com_google_protobuf//:protobuf", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index a650657de..53f7cf0c4 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -2660,7 +2660,7 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( const cel::TypeProvider& FlatExprBuilder::GetTypeProvider() const { return use_legacy_type_provider_ ? static_cast( - *GetLegacyRuntimeTypeProvider(type_registry_)) + GetLegacyRuntimeTypeProvider(type_registry_)) : GetRuntimeTypeProvider(type_registry_); } diff --git a/eval/public/BUILD b/eval/public/BUILD index 31ad2d480..7e172cd94 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -813,6 +813,7 @@ cc_library( "//eval/public/structs:legacy_type_provider", "//eval/public/structs:protobuf_descriptor_type_provider", "//runtime:type_registry", + "@com_google_absl//absl/base", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/eval/public/cel_type_registry.h b/eval/public/cel_type_registry.h index 3fb80bcea..b54b5bb99 100644 --- a/eval/public/cel_type_registry.h +++ b/eval/public/cel_type_registry.h @@ -20,6 +20,7 @@ #include #include +#include "absl/base/call_once.h" #include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" @@ -28,6 +29,7 @@ #include "base/type_provider.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_provider.h" +#include "eval/public/structs/protobuf_descriptor_type_provider.h" #include "runtime/type_registry.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -60,7 +62,9 @@ class CelTypeRegistry { CelTypeRegistry(const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory) - : modern_type_registry_(descriptor_pool, message_factory) {} + : descriptor_pool_(descriptor_pool), + message_factory_(message_factory), + modern_type_registry_(descriptor_pool, message_factory) {} ~CelTypeRegistry() = default; @@ -77,8 +81,12 @@ class CelTypeRegistry { // Get the first registered type provider. std::shared_ptr GetFirstTypeProvider() const { - return cel::runtime_internal::GetLegacyRuntimeTypeProvider( - modern_type_registry_); + absl::call_once(legacy_type_provider_once_, [&]() { + this->legacy_type_provider_ = std::make_shared< + google::api::expr::runtime::ProtobufDescriptorProvider>( + descriptor_pool_, message_factory_); + }); + return legacy_type_provider_; } // Returns the effective type provider that has been configured with the @@ -136,6 +144,13 @@ class CelTypeRegistry { } private: + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; + google::protobuf::MessageFactory* absl_nullable message_factory_; + + // Legacy type provider. This is now disconnected from the actual type + // resolution, but preserved for legacy clients that used it directly. + mutable absl::once_flag legacy_type_provider_once_; + mutable std::shared_ptr legacy_type_provider_; // Internal modern registry. cel::TypeRegistry modern_type_registry_; }; diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index 7247630bf..d8a442f34 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -184,17 +184,14 @@ cc_library( srcs = ["legacy_runtime_type_provider.cc"], hdrs = ["legacy_runtime_type_provider.h"], deps = [ + ":runtime_type_provider", "//common:legacy_value", "//common:type", "//common:value", - "//eval/public:message_wrapper", - "//eval/public/structs:legacy_type_info_apis", - "//eval/public/structs:protobuf_descriptor_type_provider", "//internal:status_macros", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", - "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", ], ) @@ -204,13 +201,12 @@ cc_test( srcs = ["legacy_runtime_type_provider_test.cc"], deps = [ ":legacy_runtime_type_provider", + ":runtime_type_provider", "//common:type", "//common:value", "//internal:testing", "//internal:testing_descriptor_pool", "//internal:testing_message_factory", - "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", ], diff --git a/runtime/internal/legacy_runtime_type_provider.cc b/runtime/internal/legacy_runtime_type_provider.cc index f6f3c26ca..9eca07652 100644 --- a/runtime/internal/legacy_runtime_type_provider.cc +++ b/runtime/internal/legacy_runtime_type_provider.cc @@ -23,24 +23,16 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "common/legacy_value.h" -#include "common/type.h" -#include "common/type_introspector.h" #include "common/value.h" #include "common/values/value_builder.h" -#include "eval/public/message_wrapper.h" -#include "eval/public/structs/legacy_type_info_apis.h" #include "internal/status_macros.h" #include "google/protobuf/arena.h" -#include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace cel::runtime_internal { namespace { -using google::api::expr::runtime::LegacyTypeInfoApis; -using google::api::expr::runtime::MessageWrapper; - class LegacyValueBuilder final : public cel::ValueBuilder { public: LegacyValueBuilder(google::protobuf::Arena* absl_nonnull arena, @@ -92,52 +84,4 @@ LegacyRuntimeTypeProvider::NewValueBuilder( return std::make_unique(arena, std::move(builder)); } -absl::StatusOr> LegacyRuntimeTypeProvider::FindTypeImpl( - absl::string_view name) const { - if (auto type = cel::FindWellKnownType(name); type.has_value()) { - return type; - } - if (auto type_info = ProvideLegacyTypeInfo(name); type_info.has_value()) { - const auto* descriptor = (*type_info)->GetDescriptor(MessageWrapper()); - if (descriptor != nullptr) { - return cel::MessageType(descriptor); - } - return cel::common_internal::MakeBasicStructType( - (*type_info)->GetTypename(MessageWrapper())); - } - return std::nullopt; -} - -absl::StatusOr> -LegacyRuntimeTypeProvider::FindStructTypeFieldByNameImpl( - absl::string_view type, absl::string_view name) const { - if (auto result = cel::FindWellKnownTypeFieldByName(type, name); - result.has_value()) { - return result; - } - std::optional type_info = - ProvideLegacyTypeInfo(type); - if (!type_info.has_value()) { - return std::nullopt; - } - if (const auto* descriptor = (*type_info)->GetDescriptor(MessageWrapper()); - descriptor != nullptr) { - // If it's a normal proto, just use the descriptor to find the field. - // Allows us to get the same optimizations as the modern value in most - // cases. - const google::protobuf::FieldDescriptor* field = descriptor->FindFieldByName(name); - if (field != nullptr) { - return cel::StructTypeField(cel::MessageTypeField(field)); - } - } - - if (auto field_desc = (*type_info)->FindFieldByName(name); - field_desc.has_value()) { - return cel::common_internal::BasicStructTypeField( - field_desc->name, field_desc->number, cel::DynType{}); - } - - return std::nullopt; -} - } // namespace cel::runtime_internal diff --git a/runtime/internal/legacy_runtime_type_provider.h b/runtime/internal/legacy_runtime_type_provider.h index ca277f3ec..2d289f3ed 100644 --- a/runtime/internal/legacy_runtime_type_provider.h +++ b/runtime/internal/legacy_runtime_type_provider.h @@ -23,23 +23,24 @@ #include "common/type.h" #include "common/type_reflector.h" #include "common/value.h" -#include "eval/public/structs/protobuf_descriptor_type_provider.h" +#include "runtime/internal/runtime_type_provider.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" namespace cel::runtime_internal { -class LegacyRuntimeTypeProvider final - : public google::api::expr::runtime::ProtobufDescriptorProvider, - public TypeReflector { +// LegacyRuntimeTypeProvider is a TypeReflector that uses a RuntimeTypeProvider +// internally to provide types with the google::api::expr::runtime::CelValue +// APIs. It prefers to create wrapped legacy values but otherwise proxies to +// the standard RuntimeTypeProvider. +class LegacyRuntimeTypeProvider final : public TypeReflector { public: LegacyRuntimeTypeProvider( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nullable message_factory) - : google::api::expr::runtime::ProtobufDescriptorProvider(descriptor_pool, - message_factory), - descriptor_pool_(descriptor_pool) {} + const RuntimeTypeProvider* absl_nonnull runtime_type_provider) + : descriptor_pool_(descriptor_pool), + runtime_type_provider_(runtime_type_provider) {} absl::StatusOr NewValueBuilder( absl::string_view name, @@ -48,13 +49,18 @@ class LegacyRuntimeTypeProvider final protected: absl::StatusOr> FindTypeImpl( - absl::string_view name) const override; + absl::string_view name) const override { + return runtime_type_provider_->FindType(name); + } absl::StatusOr> FindStructTypeFieldByNameImpl( - absl::string_view type, absl::string_view name) const override; + absl::string_view type, absl::string_view name) const override { + return runtime_type_provider_->FindStructTypeFieldByName(type, name); + } private: const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; + const RuntimeTypeProvider* absl_nonnull runtime_type_provider_; }; } // namespace cel::runtime_internal diff --git a/runtime/internal/legacy_runtime_type_provider_test.cc b/runtime/internal/legacy_runtime_type_provider_test.cc index a83f11390..bd3caddc4 100644 --- a/runtime/internal/legacy_runtime_type_provider_test.cc +++ b/runtime/internal/legacy_runtime_type_provider_test.cc @@ -22,6 +22,7 @@ #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "internal/testing_message_factory.h" +#include "runtime/internal/runtime_type_provider.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -33,8 +34,9 @@ namespace { using ::cel::expr::conformance::proto3::TestAllTypes; TEST(LegacyRuntimeTypeProviderTest, FindType) { + RuntimeTypeProvider type_provider(cel::internal::GetTestingDescriptorPool()); LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory()); + &type_provider); ASSERT_OK_AND_ASSIGN(std::optional wrapper_type, provider.FindType("google.protobuf.Int64Value")); ASSERT_TRUE(wrapper_type.has_value()); @@ -50,18 +52,18 @@ TEST(LegacyRuntimeTypeProviderTest, FindType) { } TEST(LegacyRuntimeTypeProviderTest, FindTypeNotFound) { - LegacyRuntimeTypeProvider provider( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); + RuntimeTypeProvider type_provider(google::protobuf::DescriptorPool::generated_pool()); + LegacyRuntimeTypeProvider provider(google::protobuf::DescriptorPool::generated_pool(), + &type_provider); ASSERT_OK_AND_ASSIGN(std::optional type, provider.FindType("UnknownType")); EXPECT_FALSE(type.has_value()); } TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByName) { - LegacyRuntimeTypeProvider provider( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); + RuntimeTypeProvider type_provider(google::protobuf::DescriptorPool::generated_pool()); + LegacyRuntimeTypeProvider provider(google::protobuf::DescriptorPool::generated_pool(), + &type_provider); ASSERT_OK_AND_ASSIGN(std::optional field, provider.FindStructTypeFieldByName( "google.protobuf.Int64Value", "value")); @@ -72,9 +74,9 @@ TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByName) { } TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByNameNotFound) { - LegacyRuntimeTypeProvider provider( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); + RuntimeTypeProvider type_provider(google::protobuf::DescriptorPool::generated_pool()); + LegacyRuntimeTypeProvider provider(google::protobuf::DescriptorPool::generated_pool(), + &type_provider); ASSERT_OK_AND_ASSIGN(std::optional field, provider.FindStructTypeFieldByName( "google.protobuf.Int64Value", "unknown_field")); @@ -87,8 +89,9 @@ TEST(LegacyRuntimeTypeProviderTest, FindStructTypeFieldByNameNotFound) { } TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderMessage) { + RuntimeTypeProvider type_provider(cel::internal::GetTestingDescriptorPool()); LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory()); + &type_provider); google::protobuf::Arena arena; ASSERT_OK_AND_ASSIGN(auto builder, provider.NewValueBuilder( @@ -109,8 +112,9 @@ TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderMessage) { } TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderWellKnownType) { + RuntimeTypeProvider type_provider(cel::internal::GetTestingDescriptorPool()); LegacyRuntimeTypeProvider provider(cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory()); + &type_provider); google::protobuf::Arena arena; ASSERT_OK_AND_ASSIGN(auto builder, provider.NewValueBuilder( @@ -128,9 +132,9 @@ TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderWellKnownType) { } TEST(LegacyRuntimeTypeProviderTest, NewValueBuilderNotFound) { - LegacyRuntimeTypeProvider provider( - google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory()); + RuntimeTypeProvider type_provider(google::protobuf::DescriptorPool::generated_pool()); + LegacyRuntimeTypeProvider provider(google::protobuf::DescriptorPool::generated_pool(), + &type_provider); google::protobuf::LinkMessageReflection(); google::protobuf::Arena arena; ASSERT_OK_AND_ASSIGN( diff --git a/runtime/type_registry.cc b/runtime/type_registry.cc index a1e8b0328..58bce8afd 100644 --- a/runtime/type_registry.cc +++ b/runtime/type_registry.cc @@ -35,9 +35,7 @@ TypeRegistry::TypeRegistry( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory) : type_provider_(descriptor_pool), - legacy_type_provider_( - std::make_shared( - descriptor_pool, message_factory)) { + legacy_type_provider_(descriptor_pool, &type_provider_) { RegisterEnum("google.protobuf.NullValue", {{"NULL_VALUE", 0}}); } diff --git a/runtime/type_registry.h b/runtime/type_registry.h index eadd1f1ea..22f923bd6 100644 --- a/runtime/type_registry.h +++ b/runtime/type_registry.h @@ -41,8 +41,8 @@ class TypeRegistry; namespace runtime_internal { const RuntimeTypeProvider& GetRuntimeTypeProvider( const TypeRegistry& type_registry); -const absl_nonnull std::shared_ptr& -GetLegacyRuntimeTypeProvider(const TypeRegistry& type_registry); +const LegacyRuntimeTypeProvider& GetLegacyRuntimeTypeProvider( + const TypeRegistry& type_registry); // Returns a memoized table of fully qualified enum values. // @@ -104,10 +104,9 @@ class TypeRegistry { private: friend const runtime_internal::RuntimeTypeProvider& runtime_internal::GetRuntimeTypeProvider(const TypeRegistry& type_registry); - friend const - absl_nonnull std::shared_ptr& - runtime_internal::GetLegacyRuntimeTypeProvider( - const TypeRegistry& type_registry); + friend const runtime_internal::LegacyRuntimeTypeProvider& + runtime_internal::GetLegacyRuntimeTypeProvider( + const TypeRegistry& type_registry); friend std::shared_ptr> runtime_internal::GetEnumValueTable(const TypeRegistry& type_registry); @@ -116,8 +115,7 @@ class TypeRegistry { GetEnumValueTable() const; runtime_internal::RuntimeTypeProvider type_provider_; - absl_nonnull std::shared_ptr - legacy_type_provider_; + runtime_internal::LegacyRuntimeTypeProvider legacy_type_provider_; absl::flat_hash_map enum_types_; // memoized fully qualified enumerator names. @@ -139,8 +137,8 @@ inline const RuntimeTypeProvider& GetRuntimeTypeProvider( const TypeRegistry& type_registry) { return type_registry.type_provider_; } -inline const absl_nonnull std::shared_ptr& -GetLegacyRuntimeTypeProvider(const TypeRegistry& type_registry) { +inline const LegacyRuntimeTypeProvider& GetLegacyRuntimeTypeProvider( + const TypeRegistry& type_registry) { return type_registry.legacy_type_provider_; } inline std::shared_ptr> From 035cea4364184e52d24626e83f0846d59e8746f1 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 12 Aug 2026 09:34:43 -0700 Subject: [PATCH 173/350] Restrict usages of TrivialTypeInfo to the singleton. PiperOrigin-RevId: 963494493 --- eval/public/BUILD | 7 +++++++ eval/public/structs/BUILD | 1 + eval/public/structs/trivial_legacy_type_info.h | 14 ++++++++++++-- .../structs/trivial_legacy_type_info_test.cc | 8 -------- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/eval/public/BUILD b/eval/public/BUILD index 7e172cd94..0c9c6f23e 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -29,6 +29,13 @@ licenses(["notice"]) exports_files(["LICENSE"]) +# No new users should depend on MessageWrapper using legacy APIs. +# This was added as a temporary workaround for implementing custom reflection operations +# on messages that were compiled with the LITE_RUNTIME (MessageLite) option. +# Internally, we now assume legacy Message types are always normal (Message) protobuf messages with +# standard reflection operations. +# Custom structs (including MessageLite protos) should use the modern API (see +# `cel::CustomStructValue`). cc_library( name = "message_wrapper", hdrs = [ diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 14bd607a3..82e6fd5de 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -374,6 +374,7 @@ cc_library( ":legacy_type_info_apis", "//eval/public:message_wrapper", "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/strings:string_view", ], ) diff --git a/eval/public/structs/trivial_legacy_type_info.h b/eval/public/structs/trivial_legacy_type_info.h index 2189bd478..dd413e723 100644 --- a/eval/public/structs/trivial_legacy_type_info.h +++ b/eval/public/structs/trivial_legacy_type_info.h @@ -18,6 +18,7 @@ #include #include "absl/base/no_destructor.h" +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" #include "eval/public/message_wrapper.h" #include "eval/public/structs/legacy_type_info_apis.h" @@ -27,7 +28,12 @@ namespace google::api::expr::runtime { // Implementation of type info APIs suitable for testing where no message // operations need to be supported. class TrivialTypeInfo : public LegacyTypeInfoApis { + private: + struct Key {}; + public: + explicit TrivialTypeInfo(Key&) {} + absl::string_view GetTypename(const MessageWrapper& wrapper) const override { return "opaque"; } @@ -43,10 +49,14 @@ class TrivialTypeInfo : public LegacyTypeInfoApis { return nullptr; } - static const TrivialTypeInfo* GetInstance() { - static absl::NoDestructor kInstance; + static const TrivialTypeInfo* absl_nonnull GetInstance() { + static absl::NoDestructor kKey; + static absl::NoDestructor kInstance(*kKey); return &*kInstance; } + + private: + TrivialTypeInfo() = default; }; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/trivial_legacy_type_info_test.cc b/eval/public/structs/trivial_legacy_type_info_test.cc index ea7b3977e..5001c85d5 100644 --- a/eval/public/structs/trivial_legacy_type_info_test.cc +++ b/eval/public/structs/trivial_legacy_type_info_test.cc @@ -21,35 +21,27 @@ namespace google::api::expr::runtime { namespace { TEST(TrivialTypeInfo, GetTypename) { - TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.GetTypename(wrapper), "opaque"); EXPECT_EQ(TrivialTypeInfo::GetInstance()->GetTypename(wrapper), "opaque"); } TEST(TrivialTypeInfo, DebugString) { - TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.DebugString(wrapper), "opaque"); EXPECT_EQ(TrivialTypeInfo::GetInstance()->DebugString(wrapper), "opaque"); } TEST(TrivialTypeInfo, GetAccessApis) { - TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.GetAccessApis(wrapper), nullptr); EXPECT_EQ(TrivialTypeInfo::GetInstance()->GetAccessApis(wrapper), nullptr); } TEST(TrivialTypeInfo, FindFieldByName) { - TrivialTypeInfo info; MessageWrapper wrapper; - EXPECT_EQ(info.FindFieldByName("foo"), std::nullopt); EXPECT_EQ(TrivialTypeInfo::GetInstance()->FindFieldByName("foo"), std::nullopt); } From 4e9cc59a8de411c1577edbac022f4e942798dd62 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 12 Aug 2026 14:16:08 -0700 Subject: [PATCH 174/350] Call LegacyMessage reflection accesors directly. Removes the reflected AccessApis lookup now that we enforce only a few expected implementations and can determine which implementation was intended without vtable dispatch. PiperOrigin-RevId: 963655068 --- common/BUILD | 1 + common/legacy_value.cc | 61 +++++++++--------- eval/public/structs/BUILD | 15 ++++- eval/public/structs/legacy_type_info_apis.h | 7 ++- .../structs/proto_message_type_adapter.cc | 10 +++ .../structs/proto_message_type_adapter.h | 6 ++ .../public/structs/trivial_legacy_type_info.h | 41 +----------- .../trivial_legacy_type_info_internal.h | 63 +++++++++++++++++++ 8 files changed, 128 insertions(+), 76 deletions(-) create mode 100644 eval/public/structs/trivial_legacy_type_info_internal.h diff --git a/common/BUILD b/common/BUILD index 8f0304a65..5b91f419e 100644 --- a/common/BUILD +++ b/common/BUILD @@ -799,6 +799,7 @@ cc_library( "//eval/public/structs:legacy_type_adapter", "//eval/public/structs:legacy_type_info_apis", "//eval/public/structs:proto_message_type_adapter", + "//eval/public/structs:trivial_legacy_type_info_internal", "//extensions/protobuf/internal:map_reflection", "//extensions/protobuf/internal:qualify", "//internal:casts", diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 50de9a0f8..3f3de6d85 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -55,6 +55,7 @@ #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" #include "eval/public/structs/proto_message_type_adapter.h" +#include "eval/public/structs/trivial_legacy_type_info_internal.h" #include "internal/json.h" #include "internal/status_macros.h" #include "internal/well_known_types.h" @@ -71,6 +72,7 @@ namespace cel { namespace { +using ::cel::interop_internal::TrivialTypeInfo; using ::google::api::expr::runtime::CelList; using ::google::api::expr::runtime::CelMap; using ::google::api::expr::runtime::CelValue; @@ -80,6 +82,7 @@ using ::google::api::expr::runtime::FieldBackedMapImpl; using ::google::api::expr::runtime::GetGenericProtoTypeInfoInstance; using ::google::api::expr::runtime::LegacyTypeInfoApis; using ::google::api::expr::runtime::MessageWrapper; +using ::google::api::expr::runtime::internal::GetGenericProtoAccessApisInstance; using ::google::api::expr::runtime::internal::MaybeWrapValueToMessage; absl::Status InvalidMapKeyTypeError(ValueKind kind) { @@ -885,17 +888,16 @@ absl::Status LegacyStructValue::Equal( if (auto legacy_struct_value = common_internal::AsLegacyStructValue(other); legacy_struct_value.has_value()) { auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + if (ABSL_PREDICT_FALSE(legacy_type_info_ == + TrivialTypeInfo::GetInstance())) { return absl::UnimplementedError( absl::StrCat("legacy access APIs missing for ", GetTypeName())); } auto other_message_wrapper = AsMessageWrapper(legacy_struct_value->message_ptr(), legacy_struct_value->legacy_type_info()); - *result = BoolValue{ - access_apis->IsEqualTo(message_wrapper, other_message_wrapper)}; + *result = BoolValue{GetGenericProtoAccessApisInstance().IsEqualTo( + message_wrapper, other_message_wrapper)}; return absl::OkStatus(); } if (auto struct_value = other.AsStruct(); struct_value.has_value()) { @@ -909,12 +911,12 @@ absl::Status LegacyStructValue::Equal( bool LegacyStructValue::IsZeroValue() const { auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { return false; } - return access_apis->ListFields(message_wrapper).empty(); + return GetGenericProtoAccessApisInstance() + .ListFields(message_wrapper) + .empty(); } absl::Status LegacyStructValue::GetFieldByName( @@ -923,16 +925,14 @@ absl::Status LegacyStructValue::GetFieldByName( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { *result = NoSuchFieldError(name); return absl::OkStatus(); } - CEL_ASSIGN_OR_RETURN( - auto cel_value, - access_apis->GetField(name, message_wrapper, unboxing_options, - MemoryManagerRef::Pooling(arena))); + CEL_ASSIGN_OR_RETURN(auto cel_value, + GetGenericProtoAccessApisInstance().GetField( + name, message_wrapper, unboxing_options, + MemoryManagerRef::Pooling(arena))); CEL_RETURN_IF_ERROR(ModernValue(arena, cel_value, *result)); return absl::OkStatus(); } @@ -949,12 +949,10 @@ absl::Status LegacyStructValue::GetFieldByNumber( absl::StatusOr LegacyStructValue::HasFieldByName( absl::string_view name) const { auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { return NoSuchFieldError(name).NativeValue(); } - return access_apis->HasField(name, message_wrapper); + return GetGenericProtoAccessApisInstance().HasField(name, message_wrapper); } absl::StatusOr LegacyStructValue::HasFieldByNumber(int64_t number) const { @@ -968,20 +966,19 @@ absl::Status LegacyStructValue::ForEachField( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { return absl::UnimplementedError( absl::StrCat("legacy access APIs missing for ", GetTypeName())); } - auto field_names = access_apis->ListFields(message_wrapper); + const auto& access_apis = GetGenericProtoAccessApisInstance(); + auto field_names = access_apis.ListFields(message_wrapper); Value value; for (const auto& field_name : field_names) { CEL_ASSIGN_OR_RETURN( auto cel_value, - access_apis->GetField(field_name, message_wrapper, - ProtoWrapperTypeOptions::kUnsetNull, - MemoryManagerRef::Pooling(arena))); + access_apis.GetField(field_name, message_wrapper, + ProtoWrapperTypeOptions::kUnsetNull, + MemoryManagerRef::Pooling(arena))); CEL_RETURN_IF_ERROR(ModernValue(arena, cel_value, value)); CEL_ASSIGN_OR_RETURN(auto ok, callback(field_name, value)); if (!ok) { @@ -1001,9 +998,7 @@ absl::Status LegacyStructValue::Qualify( return absl::InvalidArgumentError("invalid select qualifier path."); } auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - const auto* access_apis = - message_wrapper.legacy_type_info()->GetAccessApis(message_wrapper); - if (ABSL_PREDICT_FALSE(access_apis == nullptr)) { + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { absl::string_view field_name = absl::visit( absl::Overload( [](const FieldSpecifier& field) -> absl::string_view { @@ -1017,9 +1012,9 @@ absl::Status LegacyStructValue::Qualify( *count = -1; return absl::OkStatus(); } - CEL_ASSIGN_OR_RETURN( - auto legacy_result, - access_apis->Qualify(qualifiers, message_wrapper, presence_test, + CEL_ASSIGN_OR_RETURN(auto legacy_result, + GetGenericProtoAccessApisInstance().Qualify( + qualifiers, message_wrapper, presence_test, MemoryManager::Pooling(arena))); CEL_RETURN_IF_ERROR(ModernValue(arena, legacy_result.value, *result)); *count = legacy_result.qualifier_count; diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 82e6fd5de..992651b8b 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -367,9 +367,8 @@ cc_library( ) cc_library( - name = "trivial_legacy_type_info", - testonly = True, - hdrs = ["trivial_legacy_type_info.h"], + name = "trivial_legacy_type_info_internal", + hdrs = ["trivial_legacy_type_info_internal.h"], deps = [ ":legacy_type_info_apis", "//eval/public:message_wrapper", @@ -379,6 +378,16 @@ cc_library( ], ) +cc_library( + name = "trivial_legacy_type_info", + testonly = True, + hdrs = ["trivial_legacy_type_info.h"], + deps = [ + ":legacy_type_info_apis", + ":trivial_legacy_type_info_internal", + ], +) + cc_test( name = "trivial_legacy_type_info_test", srcs = ["trivial_legacy_type_info_test.cc"], diff --git a/eval/public/structs/legacy_type_info_apis.h b/eval/public/structs/legacy_type_info_apis.h index d7f5db3fa..999f12945 100644 --- a/eval/public/structs/legacy_type_info_apis.h +++ b/eval/public/structs/legacy_type_info_apis.h @@ -23,6 +23,10 @@ #include "eval/public/message_wrapper.h" #include "google/protobuf/descriptor.h" +namespace cel::interop_internal { +struct TrivialTypeInfo; +} // namespace cel::interop_internal + namespace google::api::expr::runtime { // Forward declared to resolve cyclic dependency. @@ -31,7 +35,6 @@ class LegacyTypeAccessApis; // Forward declare permitted subclasses. class DucktypedMessageAdapter; class ProtoMessageTypeAdapter; -class TrivialTypeInfo; // Interface for providing type info from a user defined type (represented as a // message). @@ -100,7 +103,7 @@ class LegacyTypeInfoApis { // supported using the cel::Value APIs. friend class DucktypedMessageAdapter; friend class ProtoMessageTypeAdapter; - friend class TrivialTypeInfo; + friend class cel::interop_internal::TrivialTypeInfo; LegacyTypeInfoApis() = default; }; diff --git a/eval/public/structs/proto_message_type_adapter.cc b/eval/public/structs/proto_message_type_adapter.cc index 2f72ec68a..228710c59 100644 --- a/eval/public/structs/proto_message_type_adapter.cc +++ b/eval/public/structs/proto_message_type_adapter.cc @@ -362,6 +362,10 @@ class DucktypedMessageAdapter : public LegacyTypeAccessApis, static absl::NoDestructor instance; return *instance; } + + private: + friend class absl::NoDestructor; + DucktypedMessageAdapter() = default; }; namespace { @@ -488,4 +492,10 @@ const LegacyTypeInfoApis& GetGenericProtoTypeInfoInstance() { return DucktypedMessageAdapter::GetSingleton(); } +namespace internal { +const LegacyTypeAccessApis& GetGenericProtoAccessApisInstance() { + return DucktypedMessageAdapter::GetSingleton(); +} +} // namespace internal + } // namespace google::api::expr::runtime diff --git a/eval/public/structs/proto_message_type_adapter.h b/eval/public/structs/proto_message_type_adapter.h index 867ba39cf..f1598abd9 100644 --- a/eval/public/structs/proto_message_type_adapter.h +++ b/eval/public/structs/proto_message_type_adapter.h @@ -21,6 +21,8 @@ #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" #include "common/memory.h" #include "eval/public/cel_options.h" #include "eval/public/cel_value.h" @@ -105,6 +107,10 @@ absl::StatusOr CreateCelValueFromField( // instead of expecting a particular message type given a TypeInfo. const LegacyTypeInfoApis& GetGenericProtoTypeInfoInstance(); +namespace internal { +const LegacyTypeAccessApis& GetGenericProtoAccessApisInstance(); +} // namespace internal + } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_PROTO_MESSAGE_TYPE_ADAPTER_H_ diff --git a/eval/public/structs/trivial_legacy_type_info.h b/eval/public/structs/trivial_legacy_type_info.h index dd413e723..2a1bf346b 100644 --- a/eval/public/structs/trivial_legacy_type_info.h +++ b/eval/public/structs/trivial_legacy_type_info.h @@ -15,49 +15,14 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_H_ -#include - -#include "absl/base/no_destructor.h" -#include "absl/base/nullability.h" -#include "absl/strings/string_view.h" -#include "eval/public/message_wrapper.h" -#include "eval/public/structs/legacy_type_info_apis.h" +#include "eval/public/structs/legacy_type_info_apis.h" // IWYU pragma: keep +#include "eval/public/structs/trivial_legacy_type_info_internal.h" namespace google::api::expr::runtime { // Implementation of type info APIs suitable for testing where no message // operations need to be supported. -class TrivialTypeInfo : public LegacyTypeInfoApis { - private: - struct Key {}; - - public: - explicit TrivialTypeInfo(Key&) {} - - absl::string_view GetTypename(const MessageWrapper& wrapper) const override { - return "opaque"; - } - - std::string DebugString(const MessageWrapper& wrapper) const override { - return "opaque"; - } - - const LegacyTypeAccessApis* GetAccessApis( - const MessageWrapper& wrapper) const override { - // Accessors unsupported -- caller should treat this as an opaque type (no - // fields defined, field access always results in a CEL error). - return nullptr; - } - - static const TrivialTypeInfo* absl_nonnull GetInstance() { - static absl::NoDestructor kKey; - static absl::NoDestructor kInstance(*kKey); - return &*kInstance; - } - - private: - TrivialTypeInfo() = default; -}; +using TrivialTypeInfo = cel::interop_internal::TrivialTypeInfo; } // namespace google::api::expr::runtime diff --git a/eval/public/structs/trivial_legacy_type_info_internal.h b/eval/public/structs/trivial_legacy_type_info_internal.h new file mode 100644 index 000000000..68408070a --- /dev/null +++ b/eval/public/structs/trivial_legacy_type_info_internal.h @@ -0,0 +1,63 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_INTERNAL_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_INTERNAL_H_ + +#include + +#include "absl/base/no_destructor.h" +#include "absl/base/nullability.h" +#include "absl/strings/string_view.h" +#include "eval/public/message_wrapper.h" +#include "eval/public/structs/legacy_type_info_apis.h" + +namespace cel::interop_internal { + +// Implementation details. Exported here for the sake of Legacy->Modern +// Value adaptation. +class TrivialTypeInfo : public google::api::expr::runtime::LegacyTypeInfoApis { + public: + absl::string_view GetTypename( + const google::api::expr::runtime::MessageWrapper& wrapper) + const override { + return "opaque"; + } + + std::string DebugString(const google::api::expr::runtime::MessageWrapper& + wrapper) const override { + return "opaque"; + } + + const google::api::expr::runtime::LegacyTypeAccessApis* GetAccessApis( + const google::api::expr::runtime::MessageWrapper& wrapper) + const override { + // Accessors unsupported -- caller should treat this as an opaque type (no + // fields defined, field access always results in a CEL error). + return nullptr; + } + + static const TrivialTypeInfo* absl_nonnull GetInstance() { + static absl::NoDestructor kInstance; + return &*kInstance; + } + + private: + friend class absl::NoDestructor; + TrivialTypeInfo() = default; +}; + +} // namespace cel::interop_internal + +#endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_TRIVIAL_LEGACY_TYPE_INFO_INTERNAL_H_ From ad9ebbe3c14602c79debda5b1906562c51fd8b27 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 12 Aug 2026 16:44:07 -0700 Subject: [PATCH 175/350] Migrate low-touch cel::LegacyStructValue member functions to use the modern equivalent implementations. These members should be safer than the getters (either non-allocating or not a hot path). PiperOrigin-RevId: 963730307 --- common/legacy_value.cc | 103 ++++++++---------- eval/public/structs/BUILD | 1 + .../proto_message_type_adapter_test.cc | 9 +- 3 files changed, 47 insertions(+), 66 deletions(-) diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 3f3de6d85..184a09c0b 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -850,34 +850,30 @@ absl::Status LegacyStructValue::ConvertToJson( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Message* absl_nonnull json) const { + ABSL_DCHECK(message_ptr_ != nullptr); ABSL_DCHECK(descriptor_pool != nullptr); ABSL_DCHECK(message_factory != nullptr); ABSL_DCHECK(json != nullptr); ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE); - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - - return internal::MessageToJson( - *google::protobuf::DownCastMessage(message_wrapper.message_ptr()), - descriptor_pool, message_factory, json); + return internal::MessageToJson(*message_ptr_, descriptor_pool, + message_factory, json); } absl::Status LegacyStructValue::ConvertToJsonObject( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Message* absl_nonnull json) const { + ABSL_DCHECK(message_ptr_ != nullptr); ABSL_DCHECK(descriptor_pool != nullptr); ABSL_DCHECK(message_factory != nullptr); ABSL_DCHECK(json != nullptr); ABSL_DCHECK_EQ(json->GetDescriptor()->well_known_type(), google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT); - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - - return internal::MessageToJson( - *google::protobuf::DownCastMessage(message_wrapper.message_ptr()), - descriptor_pool, message_factory, json); + return internal::MessageToJson(*message_ptr_, descriptor_pool, + message_factory, json); } absl::Status LegacyStructValue::Equal( @@ -885,38 +881,36 @@ absl::Status LegacyStructValue::Equal( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { - if (auto legacy_struct_value = common_internal::AsLegacyStructValue(other); - legacy_struct_value.has_value()) { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); - if (ABSL_PREDICT_FALSE(legacy_type_info_ == - TrivialTypeInfo::GetInstance())) { - return absl::UnimplementedError( - absl::StrCat("legacy access APIs missing for ", GetTypeName())); - } - auto other_message_wrapper = - AsMessageWrapper(legacy_struct_value->message_ptr(), - legacy_struct_value->legacy_type_info()); - *result = BoolValue{GetGenericProtoAccessApisInstance().IsEqualTo( - message_wrapper, other_message_wrapper)}; - return absl::OkStatus(); + ABSL_DCHECK(message_ptr_ != nullptr); + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { + return absl::UnimplementedError( + absl::StrCat("legacy access APIs missing for ", GetTypeName())); } - if (auto struct_value = other.AsStruct(); struct_value.has_value()) { - return common_internal::StructValueEqual( - common_internal::LegacyStructValue(message_ptr_, legacy_type_info_), - *struct_value, descriptor_pool, message_factory, arena, result); + auto modern_value = UnsafeParsedMessageValue(message_ptr_); + + // Unwrap the rhs if it's a legacy struct so the normal implementation + // doesn't hit the fallback abstract struct comparison. + if (auto other_legacy = common_internal::AsLegacyStructValue(other); + other_legacy) { + if (other_legacy->legacy_type_info_ == TrivialTypeInfo::GetInstance()) { + return absl::UnimplementedError(absl::StrCat( + "legacy access APIs missing for ", other_legacy->GetTypeName())); + } + auto other_message = UnsafeParsedMessageValue(other_legacy->message_ptr_); + return modern_value.Equal(other_message, descriptor_pool, message_factory, + arena, result); } - *result = FalseValue(); - return absl::OkStatus(); + + return modern_value.Equal(other, descriptor_pool, message_factory, arena, + result); } bool LegacyStructValue::IsZeroValue() const { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); + ABSL_DCHECK(message_ptr_ != nullptr); if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { return false; } - return GetGenericProtoAccessApisInstance() - .ListFields(message_wrapper) - .empty(); + return UnsafeParsedMessageValue(message_ptr_).IsZeroValue(); } absl::Status LegacyStructValue::GetFieldByName( @@ -948,16 +942,19 @@ absl::Status LegacyStructValue::GetFieldByNumber( absl::StatusOr LegacyStructValue::HasFieldByName( absl::string_view name) const { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); + ABSL_DCHECK(message_ptr_ != nullptr); if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { - return NoSuchFieldError(name).NativeValue(); + return NoSuchFieldError(name).ToStatus(); } - return GetGenericProtoAccessApisInstance().HasField(name, message_wrapper); + return UnsafeParsedMessageValue(message_ptr_).HasFieldByName(name); } absl::StatusOr LegacyStructValue::HasFieldByNumber(int64_t number) const { - return absl::UnimplementedError( - "access to fields by numbers is not available for legacy structs"); + ABSL_DCHECK(message_ptr_ != nullptr); + if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { + return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + } + return UnsafeParsedMessageValue(message_ptr_).HasFieldByNumber(number); } absl::Status LegacyStructValue::ForEachField( @@ -965,27 +962,13 @@ absl::Status LegacyStructValue::ForEachField( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { - auto message_wrapper = AsMessageWrapper(message_ptr_, legacy_type_info_); + ABSL_DCHECK(message_ptr_ != nullptr); if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { return absl::UnimplementedError( absl::StrCat("legacy access APIs missing for ", GetTypeName())); } - const auto& access_apis = GetGenericProtoAccessApisInstance(); - auto field_names = access_apis.ListFields(message_wrapper); - Value value; - for (const auto& field_name : field_names) { - CEL_ASSIGN_OR_RETURN( - auto cel_value, - access_apis.GetField(field_name, message_wrapper, - ProtoWrapperTypeOptions::kUnsetNull, - MemoryManagerRef::Pooling(arena))); - CEL_RETURN_IF_ERROR(ModernValue(arena, cel_value, value)); - CEL_ASSIGN_OR_RETURN(auto ok, callback(field_name, value)); - if (!ok) { - break; - } - } - return absl::OkStatus(); + return UnsafeParsedMessageValue(message_ptr_) + .ForEachField(callback, descriptor_pool, message_factory, arena); } absl::Status LegacyStructValue::Qualify( @@ -1291,12 +1274,12 @@ const google::protobuf::Message* absl_nullable GetLegacyMessage(const Value& val auto legacy = common_internal::GetLegacyStructValue(value); const auto* legacy_type_info = legacy.legacy_type_info(); - if (legacy_type_info == nullptr) { - return nullptr; - } - if (legacy_type_info != &GetGenericProtoTypeInfoInstance()) { + if (legacy_type_info == nullptr || + legacy_type_info == TrivialTypeInfo::GetInstance()) { return nullptr; } + // This should not be possible using the normal public APIs, but possible + // if someone used the MessageWrapper class directly. if (IsWellKnownMessageType(legacy.message_ptr()->GetDescriptor())) { return nullptr; } diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 992651b8b..468867294 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -305,6 +305,7 @@ cc_test( ":legacy_type_adapter", ":legacy_type_info_apis", ":proto_message_type_adapter", + ":trivial_legacy_type_info", "//base:attributes", "//common:value", "//common:value_testing", diff --git a/eval/public/structs/proto_message_type_adapter_test.cc b/eval/public/structs/proto_message_type_adapter_test.cc index a74a32cb4..529052025 100644 --- a/eval/public/structs/proto_message_type_adapter_test.cc +++ b/eval/public/structs/proto_message_type_adapter_test.cc @@ -28,6 +28,7 @@ #include "eval/public/message_wrapper.h" #include "eval/public/structs/legacy_type_adapter.h" #include "eval/public/structs/legacy_type_info_apis.h" +#include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/public/testing/matchers.h" #include "eval/testutil/test_message.pb.h" #include "extensions/protobuf/memory_manager.h" @@ -1147,17 +1148,13 @@ TEST(ProtoMesssageTypeAdapter, QualifyMapIndexLeafWrongType) { HasSubstr("Invalid map key type")))))); } -TEST(ProtoMesssageTypeAdapter, InteropUnwrappingNotGeneric) { +TEST(ProtoMesssageTypeAdapter, InteropUnwrappingNotTestInstance) { google::protobuf::Arena arena; - ProtoMessageTypeAdapter adapter( - google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( - "google.api.expr.runtime.TestMessage"), - google::protobuf::MessageFactory::generated_factory()); TestMessage message; message.set_string_value("hello"); auto legacy_value = CelValue::CreateMessageWrapper( - CelValue::MessageWrapper(&message, &adapter)); + CelValue::MessageWrapper(&message, TrivialTypeInfo::GetInstance())); cel::Value modern_value; ASSERT_THAT(cel::ModernValue(&arena, legacy_value, modern_value), IsOk()); auto unwrapped = cel::interop_internal::GetLegacyMessage(modern_value); From 9bcb00a82f1c12c9df4ee46df556af8ad1e3b173 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 12 Aug 2026 17:52:58 -0700 Subject: [PATCH 176/350] [Pratt Parser] Add support for enriched source info PiperOrigin-RevId: 963760554 --- parser/BUILD | 3 ++ parser/internal/BUILD | 1 + parser/internal/pratt_parser.cc | 12 +++++- parser/internal/pratt_parser.h | 8 +++- parser/internal/pratt_parser_worker.cc | 23 +++++++++--- parser/internal/pratt_parser_worker.h | 52 +++++++++++++++++++++----- parser/parser.cc | 15 +++++--- parser/parser_test.cc | 16 +++++--- parser/source_factory.h | 18 ++++++--- 9 files changed, 114 insertions(+), 34 deletions(-) diff --git a/parser/BUILD b/parser/BUILD index c548967eb..383d82e50 100644 --- a/parser/BUILD +++ b/parser/BUILD @@ -164,6 +164,9 @@ cc_library( hdrs = [ "source_factory.h", ], + deps = [ + "@com_google_absl//absl/container:flat_hash_map", + ], ) cc_library( diff --git a/parser/internal/BUILD b/parser/internal/BUILD index 0cfb042de..8d6a85a55 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -117,6 +117,7 @@ cc_library( "//parser:macro_registry", "//parser:options", "//parser:parser_interface", + "//parser:source_factory", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/cleanup", diff --git a/parser/internal/pratt_parser.cc b/parser/internal/pratt_parser.cc index 3f59e5065..b5cf10457 100644 --- a/parser/internal/pratt_parser.cc +++ b/parser/internal/pratt_parser.cc @@ -44,6 +44,7 @@ #include "parser/macro_registry.h" #include "parser/options.h" #include "parser/parser_interface.h" +#include "parser/source_factory.h" namespace cel::parser_internal { @@ -173,7 +174,8 @@ absl::StatusOr> PrattParserImpl::PrepareSourceImpl( absl::StatusOr> PrattParseImpl( const cel::Source& source, const cel::MacroRegistry& registry, - const ParserOptions& options, std::vector* parse_issues) { + const ParserOptions& options, std::vector* parse_issues, + cel::EnrichedSourceInfo* enriched_source_info) { if (source.content().size() > options.expression_size_codepoint_limit) { return absl::InvalidArgumentError(absl::StrFormat( "expression size exceeds codepoint limit. input size: %zu, limit: %d", @@ -181,7 +183,9 @@ absl::StatusOr> PrattParseImpl( } std::vector issues; AstFactory factory(®istry); - PrattParserWorker worker(source, options, &issues, factory); + PrattParserWorker worker( + source, options, &issues, factory, + /*track_node_ranges=*/enriched_source_info != nullptr); Expr expr = worker.Parse(); if (worker.is_recursion_limit_exceeded()) { return absl::CancelledError( @@ -203,6 +207,10 @@ absl::StatusOr> PrattParseImpl( return absl::InvalidArgumentError(err_msg); } + if (enriched_source_info != nullptr) { + *enriched_source_info = cel::EnrichedSourceInfo(worker.GetNodeRanges()); + } + cel::SourceInfo source_info; source_info.set_location(std::string(source.description())); for (const auto& [id, pos] : worker.GetNodePositions()) { diff --git a/parser/internal/pratt_parser.h b/parser/internal/pratt_parser.h index 006a10c28..a33ca803b 100644 --- a/parser/internal/pratt_parser.h +++ b/parser/internal/pratt_parser.h @@ -25,6 +25,7 @@ #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "common/ast.h" #include "common/source.h" #include "parser/macro.h" @@ -32,6 +33,10 @@ #include "parser/options.h" #include "parser/parser_interface.h" +namespace cel { +class EnrichedSourceInfo; +} // namespace cel + namespace cel::parser_internal { // PrattParserImpl implements the Pratt parsing algorithm for CEL expressions. @@ -73,7 +78,8 @@ class PrattParserImpl final : public cel::Parser { absl::StatusOr> PrattParseImpl( const cel::Source& source, const cel::MacroRegistry& registry, const ParserOptions& options, - std::vector* parse_issues = nullptr); + std::vector* parse_issues = nullptr, + cel::EnrichedSourceInfo* enriched_source_info = nullptr); class PrattParserBuilderImpl final : public cel::ParserBuilder { public: diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index 3f842ea80..86319a853 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -19,6 +19,7 @@ #include #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -101,11 +102,13 @@ const BinaryOpInfo& GetBinaryOpInfo(TokenType type) { ParserWorker::ParserWorker( const cel::Source& source, const cel::ParserOptions& options, - std::vector* absl_nullable parse_issues) + std::vector* absl_nullable parse_issues, + bool track_node_ranges) : source_(source), options_(options), lexer_(source_), - parse_issues_(parse_issues) {} + parse_issues_(parse_issues), + track_node_ranges_(track_node_ranges) {} void ParserWorker::InitTokenStream() { current_token_ = Token{.type = TokenType::kError, .start = 0, .end = 0}; @@ -204,14 +207,15 @@ int64_t ParserWorker::NextId(int32_t position) { } if (position >= 0) { positions_.insert({id, position}); + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + node_ranges_.insert({id, {position, position}}); + } } return id; } int64_t ParserWorker::NextId() { return NextId(-1); } -bool ParserWorker::NodeLimitExceeded() { return node_limit_exceeded_; } - int64_t ParserWorker::CopyId(int64_t id) { if (id == 0) { return 0; @@ -220,11 +224,20 @@ int64_t ParserWorker::CopyId(int64_t id) { if (auto it = positions_.find(id); it != positions_.end()) { pos = it->second; } - return NextId(pos); + int64_t new_id = NextId(pos); + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + if (auto it = node_ranges_.find(id); it != node_ranges_.end()) { + node_ranges_[new_id] = it->second; + } + } + return new_id; } void ParserWorker::EraseId(int64_t id) { positions_.erase(id); + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + node_ranges_.erase(id); + } if (next_id_ == id + 1) { --next_id_; } diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 30fdce882..35e9ff52b 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -51,11 +51,16 @@ namespace cel::parser_internal { class ParserWorker { public: ParserWorker(const cel::Source& source, const cel::ParserOptions& options, - std::vector* absl_nullable parse_issues); + std::vector* absl_nullable parse_issues, + bool track_node_ranges = false); const absl::flat_hash_map& GetNodePositions() const { return positions_; } + const absl::flat_hash_map>& + GetNodeRanges() const { + return node_ranges_; + } absl::Span GetLineOffsets() const { return source_.line_offsets(); } @@ -75,11 +80,25 @@ class ParserWorker { // ID and Position tracking int64_t NextId(int32_t position); - int64_t NextId(const Token& token) { return NextId(token.start); } + int64_t NextId(const Token& token) { + int64_t id = NextId(token.start); + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + if (token.start >= 0 && token.end > token.start) { + node_ranges_[id] = {token.start, token.end - 1}; + } + } + return id; + } int64_t NextId(); - bool NodeLimitExceeded(); int64_t CopyId(int64_t id); void EraseId(int64_t id); + void SetNodeRange(int64_t id, int32_t begin, int32_t end) { + if (ABSL_PREDICT_FALSE(track_node_ranges_)) { + if (id != 0 && begin >= 0 && end >= begin) { + node_ranges_[id] = {begin, end}; + } + } + } // Error reporting and recovery bool is_recovery_limit_exceeded() const { @@ -100,10 +119,12 @@ class ParserWorker { int64_t next_id_ = 1; bool node_limit_exceeded_ = false; absl::flat_hash_map positions_; + absl::flat_hash_map> node_ranges_; std::vector* absl_nullable parse_issues_; int error_count_ = 0; bool lexer_error_reported_ = false; bool recursion_limit_exceeded_ = false; + bool track_node_ranges_ = false; }; struct BinaryOpInfo { @@ -131,8 +152,10 @@ class PrattParserWorker : public ParserWorker { explicit PrattParserWorker( const cel::Source& source, const cel::ParserOptions& options, std::vector* absl_nullable parse_issues, - AstFactoryInterface& ast_factory) - : ParserWorker(source, options, parse_issues), ast_factory_(ast_factory) { + AstFactoryInterface& ast_factory, + bool track_node_ranges = false) + : ParserWorker(source, options, parse_issues, track_node_ranges), + ast_factory_(ast_factory) { this->InitTokenStream(); } @@ -696,7 +719,9 @@ ExprNode PrattParserWorker::ParseList() { break; } } - Expect(TokenType::kRightBracket, "expected ']'"); + if (Expect(TokenType::kRightBracket, "expected ']'")) { + SetNodeRange(list_id, open_tok.start, current_token_.end - 1); + } return builder.Build(); } @@ -732,7 +757,9 @@ ExprNode PrattParserWorker::ParseMap() { break; } } - Expect(TokenType::kRightBrace, "expected '}'"); + if (Expect(TokenType::kRightBrace, "expected '}'")) { + SetNodeRange(map_id, open_tok.start, current_token_.end - 1); + } return builder.Build(); } @@ -774,7 +801,14 @@ ExprNode PrattParserWorker::ParseStruct( break; } } - Expect(TokenType::kRightBrace, "expected '}'"); + if (Expect(TokenType::kRightBrace, "expected '}'")) { + int32_t start_pos = open_tok.start; + auto it = positions_.find(obj_id); + if (it != positions_.end()) { + start_pos = it->second; + } + SetNodeRange(obj_id, start_pos, current_token_.end - 1); + } return builder.Build(); } @@ -1052,7 +1086,7 @@ std::optional PrattParserWorker::TryExpandMacro( if (!expander) { return std::nullopt; } - if (NodeLimitExceeded()) { + if (node_limit_exceeded_) { ReportError(expr_id, "could not expand macro: expression node limit exceeded"); return std::nullopt; diff --git a/parser/parser.cc b/parser/parser.cc index 28923e02d..9f94c3a91 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -1445,7 +1444,8 @@ cel::SourceInfo ParserVisitor::GetSourceInfo() { } EnrichedSourceInfo ParserVisitor::enriched_source_info() const { - std::map> offsets; + absl::flat_hash_map> offsets; + offsets.reserve(factory_.positions().size()); for (const auto& positions : factory_.positions()) { offsets.insert( std::pair{positions.first, @@ -1951,14 +1951,17 @@ absl::StatusOr EnrichedParse( const ParserOptions& options) { ParsedExpr parsed_expr; if (options.enable_pratt_parser) { - CEL_ASSIGN_OR_RETURN( - std::unique_ptr ast, - cel::parser_internal::PrattParseImpl(source, registry, options)); + EnrichedSourceInfo enriched_source_info; + CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, + cel::parser_internal::PrattParseImpl( + source, registry, options, + /*parse_issues=*/nullptr, &enriched_source_info)); CEL_RETURN_IF_ERROR(cel::ast_internal::ExprToProto( ast->root_expr(), parsed_expr.mutable_expr())); CEL_RETURN_IF_ERROR(cel::ast_internal::SourceInfoToProto( ast->source_info(), parsed_expr.mutable_source_info())); - return VerboseParsedExpr(std::move(parsed_expr), EnrichedSourceInfo()); + return VerboseParsedExpr(std::move(parsed_expr), + std::move(enriched_source_info)); } CEL_ASSIGN_OR_RETURN(ParseResult parse_result, ParseImpl(source, registry, options)); diff --git a/parser/parser_test.cc b/parser/parser_test.cc index c04d819b2..a7a2eb442 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1650,8 +1650,13 @@ class LocationAdorner : public cel::ExpressionAdorner { std::string ConvertEnrichedSourceInfoToString( const EnrichedSourceInfo& enriched_source_info) { + std::vector>> sorted_offsets( + enriched_source_info.offsets().begin(), + enriched_source_info.offsets().end()); + absl::c_sort(sorted_offsets); std::vector offsets; - for (const auto& offset : enriched_source_info.offsets()) { + offsets.reserve(sorted_offsets.size()); + for (const auto& offset : sorted_offsets) { offsets.push_back(absl::StrFormat( "[%d,%d,%d]", offset.first, offset.second.first, offset.second.second)); } @@ -1747,24 +1752,23 @@ TEST_P(ExpressionTest, Parse) { } } -TEST(ExpressionTest, CompositeExpressionOffsets) { - ParserOptions options; +TEST_P(ExpressionTest, CompositeExpressionOffsets) { std::vector macros = Macro::AllMacros(); std::string list_expr = "[1, 2]"; - auto list_result = EnrichedParse(list_expr, macros, "", options); + auto list_result = EnrichedParse(list_expr, macros, "", options_); ASSERT_THAT(list_result, IsOk()); auto list_offsets = list_result->enriched_source_info().offsets(); EXPECT_EQ(list_offsets.at(1), std::make_pair(0, 5)); std::string map_expr = "{'a': 1}"; - auto map_result = EnrichedParse(map_expr, macros, "", options); + auto map_result = EnrichedParse(map_expr, macros, "", options_); ASSERT_THAT(map_result, IsOk()); auto map_offsets = map_result->enriched_source_info().offsets(); EXPECT_EQ(map_offsets.at(1), std::make_pair(0, 7)); std::string msg_expr = "Msg{f: 1}"; - auto msg_result = EnrichedParse(msg_expr, macros, "", options); + auto msg_result = EnrichedParse(msg_expr, macros, "", options_); ASSERT_THAT(msg_result, IsOk()); auto msg_offsets = msg_result->enriched_source_info().offsets(); EXPECT_EQ(msg_offsets.at(1), std::make_pair(0, 8)); diff --git a/parser/source_factory.h b/parser/source_factory.h index 501e1017a..11f64be6c 100644 --- a/parser/source_factory.h +++ b/parser/source_factory.h @@ -16,15 +16,16 @@ #define THIRD_PARTY_CEL_CPP_PARSER_SOURCE_FACTORY_H_ #include -#include #include -namespace google::api::expr::parser { +#include "absl/container/flat_hash_map.h" + +namespace cel { class EnrichedSourceInfo { public: explicit EnrichedSourceInfo( - std::map> offsets) + absl::flat_hash_map> offsets) : offsets_(std::move(offsets)) {} EnrichedSourceInfo() = default; @@ -33,15 +34,22 @@ class EnrichedSourceInfo { EnrichedSourceInfo(EnrichedSourceInfo&& other) = default; EnrichedSourceInfo& operator=(EnrichedSourceInfo&& other) = default; - const std::map>& offsets() const { + const absl::flat_hash_map>& offsets() + const { return offsets_; } private: // A map between node_id and pair of start position and end position - std::map> offsets_; + absl::flat_hash_map> offsets_; }; +} // namespace cel + +namespace google::api::expr::parser { + +using EnrichedSourceInfo = ::cel::EnrichedSourceInfo; + } // namespace google::api::expr::parser #endif // THIRD_PARTY_CEL_CPP_PARSER_SOURCE_FACTORY_H_ From 1d443e4249188da5fbd0aa98a9a10c8cda7073b3 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 13 Aug 2026 16:47:14 -0700 Subject: [PATCH 177/350] Enable conformance tests for lists_ext PiperOrigin-RevId: 964365868 --- MODULE.bazel | 8 +++++++- conformance/BUILD | 5 +++++ conformance/service.cc | 8 +++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 090309f2a..b013fb86e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -64,9 +64,15 @@ bazel_dep( ) bazel_dep( name = "cel-spec", - version = "0.25.1", + version = "0.25.2", repo_name = "com_google_cel_spec", ) +git_override( + module_name = "cel-spec", + commit = "ba58ae5007845f3a1279b488cdeb79645ce958bb", + remote = "https://github.com/cel-expr/cel-spec", +) + bazel_dep( name = "platforms", version = "1.1.0", diff --git a/conformance/BUILD b/conformance/BUILD index a5889f83b..b4ac3718f 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -47,6 +47,7 @@ cc_library( "//extensions:comprehensions_v2_functions", "//extensions:comprehensions_v2_macros", "//extensions:encoders", + "//extensions:lists_functions", "//extensions:math_ext", "//extensions:math_ext_decls", "//extensions:math_ext_macros", @@ -143,6 +144,7 @@ _ALL_TESTS = [ "@com_google_cel_spec//tests/simple:testdata/fp_math.textproto", "@com_google_cel_spec//tests/simple:testdata/integer_math.textproto", "@com_google_cel_spec//tests/simple:testdata/lists.textproto", + "@com_google_cel_spec//tests/simple:testdata/lists_ext.textproto", "@com_google_cel_spec//tests/simple:testdata/logic.textproto", "@com_google_cel_spec//tests/simple:testdata/macros.textproto", "@com_google_cel_spec//tests/simple:testdata/macros2.textproto", @@ -228,6 +230,9 @@ _TESTS_TO_SKIP_LEGACY = _TESTS_TO_SKIP + [ "block_ext/basic/optional_map", "block_ext/basic/optional_map_chained", "block_ext/basic/optional_message", + + # lists_ext not implemented for legacy evaluator + "lists_ext", ] _TESTS_TO_SKIP_LEGACY_DASHBOARD = [ diff --git a/conformance/service.cc b/conformance/service.cc index 8a456a9f4..c826d4572 100644 --- a/conformance/service.cc +++ b/conformance/service.cc @@ -60,6 +60,7 @@ #include "extensions/comprehensions_v2_functions.h" #include "extensions/comprehensions_v2_macros.h" #include "extensions/encoders.h" +#include "extensions/lists_functions.h" #include "extensions/math_ext.h" #include "extensions/math_ext_decls.h" #include "extensions/math_ext_macros.h" @@ -147,6 +148,7 @@ absl::Status LegacyParse(const conformance::v1alpha1::ParseRequest& request, CEL_RETURN_IF_ERROR(cel::extensions::RegisterBindingsMacros(macros, options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterMathMacros(macros, options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterProtoMacros(macros, options)); + CEL_RETURN_IF_ERROR(cel::extensions::RegisterListsMacros(macros, options)); CEL_RETURN_IF_ERROR(cel::test::RegisterTestMacros(macros)); CEL_ASSIGN_OR_RETURN(auto source, cel::NewSource(request.cel_source(), request.source_location())); @@ -192,6 +194,8 @@ absl::Status CheckImpl(google::protobuf::Arena* arena, builder->AddLibrary(cel::extensions::EncodersCheckerLibrary())); CEL_RETURN_IF_ERROR( builder->AddLibrary(cel::extensions::ComprehensionsV2CheckerLibrary())); + CEL_RETURN_IF_ERROR( + builder->AddLibrary(cel::extensions::ListsCheckerLibrary())); } for (const auto& decl : request.type_env()) { @@ -201,7 +205,7 @@ absl::Status CheckImpl(google::protobuf::Arena* arena, auto fn_decl, cel::FunctionDeclFromV1Alpha1Proto( name, decl.function(), google::protobuf::DescriptorPool::generated_pool(), arena)); - CEL_RETURN_IF_ERROR(builder->AddFunction(std::move(fn_decl))); + CEL_RETURN_IF_ERROR(builder->MergeFunction(std::move(fn_decl))); } else if (decl.has_ident()) { CEL_ASSIGN_OR_RETURN( auto var_decl, cel::VariableDeclFromV1Alpha1Proto( @@ -536,6 +540,8 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface { builder.function_registry(), options)); CEL_RETURN_IF_ERROR(cel::extensions::RegisterMathExtensionFunctions( builder.function_registry(), options)); + CEL_RETURN_IF_ERROR(cel::extensions::RegisterListsFunctions( + builder.function_registry(), options)); return std::move(builder).Build(); } From d686b67e1a8055462a70cc5263a1fb3e08dc037a Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 14 Aug 2026 22:38:30 -0700 Subject: [PATCH 178/350] [Pratt Parser] Avoid incrementing recursion depth for non-functional parens PiperOrigin-RevId: 965063128 --- parser/internal/BUILD | 1 + parser/internal/lexer.h | 18 ++++++ parser/internal/lexer_test.cc | 37 ++++++++++++ parser/internal/pratt_parser_test.cc | 64 ++++++++++++++++++++ parser/internal/pratt_parser_worker.cc | 4 +- parser/internal/pratt_parser_worker.h | 83 +++++++++++++++++++++++++- parser/parser_test.cc | 10 ++-- 7 files changed, 207 insertions(+), 10 deletions(-) diff --git a/parser/internal/BUILD b/parser/internal/BUILD index 8d6a85a55..cdc73288a 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -93,6 +93,7 @@ cc_library( "//parser:parser_interface", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h index ef166d4bb..c35dc08e2 100644 --- a/parser/internal/lexer.h +++ b/parser/internal/lexer.h @@ -140,6 +140,13 @@ class Lexer final { std::numeric_limits::max())); } + struct Position final { + int32_t position = 0; + bool at_end = false; + bool done = false; + LexerError error; + }; + Lexer(const Lexer&) = delete; Lexer(Lexer&&) = delete; Lexer& operator=(const Lexer&) = delete; @@ -158,6 +165,17 @@ class Lexer final { [[nodiscard]] int32_t GetPosition() const { return position_; } + [[nodiscard]] Position SavePosition() const { + return Position{position_, at_end_, done_, error_}; + } + + void RestorePosition(const Position& position) { + position_ = position.position; + at_end_ = position.at_end; + done_ = position.done; + error_ = position.error; + } + private: [[nodiscard]] bool Match(char32_t c) const { return position_ < content_.size() && content_.at(position_) == c; diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc index ca311baf8..ebe0011d3 100644 --- a/parser/internal/lexer_test.cc +++ b/parser/internal/lexer_test.cc @@ -495,5 +495,42 @@ TEST(LexerErrorRecoveryTest, ResumesAfterError) { EXPECT_EQ(token.end, 6); } +TEST(LexerPositionTest, SaveAndRestorePosition) { + ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("foo + bar * 42")); + Lexer lexer(*source); + + Token tok1 = lexer.Lex(); + EXPECT_EQ(tok1.type, TokenType::kIdent); + + Token tok2 = lexer.Lex(); + EXPECT_EQ(tok2.type, TokenType::kWhitespace); + + // Save position before '+' + Lexer::Position saved = lexer.SavePosition(); + + Token tok3 = lexer.Lex(); + EXPECT_EQ(tok3.type, TokenType::kPlus); + + Token tok4 = lexer.Lex(); + EXPECT_EQ(tok4.type, TokenType::kWhitespace); + + Token tok5 = lexer.Lex(); + EXPECT_EQ(tok5.type, TokenType::kIdent); + + // Restore position to before '+' + lexer.RestorePosition(saved); + + Token tok3_restored = lexer.Lex(); + EXPECT_EQ(tok3_restored.type, TokenType::kPlus); + EXPECT_EQ(tok3_restored.start, tok3.start); + EXPECT_EQ(tok3_restored.end, tok3.end); + + Token tok4_restored = lexer.Lex(); + EXPECT_EQ(tok4_restored.type, TokenType::kWhitespace); + + Token tok5_restored = lexer.Lex(); + EXPECT_EQ(tok5_restored.type, TokenType::kIdent); +} + } // namespace } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 539372e51..8ebe5123e 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -180,6 +180,32 @@ MATCHER_P(AstIs, expected_ast, "") { return false; } +MATCHER_P(AstEq, expected_expr, "") { + KindAndIdAdorner kind_and_id_adorner; + cel::ExprPrinter printer(kind_and_id_adorner); + ParserOptions options; + auto actual_ast = Parse(arg, options); + if (!actual_ast.ok()) { + *result_listener << "\n Actual expression failed to parse: " + << actual_ast.status(); + return false; + } + std::string actual = Unindent(printer.Print((*actual_ast)->root_expr())); + auto expected_ast = Parse(expected_expr, options); + if (!expected_ast.ok()) { + *result_listener << "\n Expected expression failed to parse: " + << expected_ast.status(); + return false; + } + std::string expected = Unindent(printer.Print((*expected_ast)->root_expr())); + if (actual == expected) { + return true; + } + *result_listener << "\n Actual: " << actual + << "\n Expected: " << expected; + return false; +} + TEST_P(PrattParserTest, Parse) { const TestCase& test_case = GetParam(); cel::ParserOptions options; @@ -1472,6 +1498,44 @@ TEST(PrattParserRecursionDepthTest, ParseRecursionDepth) { StatusIs(absl::StatusCode::kCancelled)); } +TEST(PrattParserRecursionDepthTest, ParseRecursionDepthIgnoreExtraParens) { + cel::ParserOptions options; + options.max_recursion_depth = 1; + EXPECT_THAT(Parse("((((1))))", options), IsOkAndHolds(NotNull())); +} + +TEST(PrattParserRecursionDepthTest, DeeplyNestedParens) { + cel::ParserOptions options; + options.max_recursion_depth = 1; + std::string literal_expr = + std::string(1000, '(') + "42" + std::string(1000, ')'); + EXPECT_THAT(Parse(literal_expr, options), IsOkAndHolds(NotNull())); + + std::string binary_expr = + std::string(1000, '(') + "1 + 2" + std::string(1000, ')'); + EXPECT_THAT(Parse(binary_expr, options), IsOkAndHolds(NotNull())); +} + +TEST(PrattParserRecursionDepthTest, NestedAndGroupingParensCombinations) { + EXPECT_THAT("(( (1) + 2 ))", AstEq("1 + 2")); + EXPECT_THAT("((1 + 2) * (3 + 4))", AstEq("(1 + 2) * (3 + 4)")); + EXPECT_THAT("((((1)) + ((2))))", AstEq("1 + 2")); + EXPECT_THAT("(((1 + 2) * 3) + 4)", AstEq("(1 + 2) * 3 + 4")); + EXPECT_THAT("f((((1))), (((2))))", AstEq("f(1, 2)")); + EXPECT_THAT("[{((1)): ((2))}]", AstEq("[{1: 2}]")); + EXPECT_THAT("(((a))).b[0]", AstEq("a.b[0]")); +} + +TEST(PrattParserRecursionDepthTest, MismatchedParensStillReportErrors) { + cel::ParserOptions options; + EXPECT_THAT(Parse("((((1))", options), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(Parse("(((1 + 2]", options), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(Parse("(( [ 1 ) ] ))", options), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + TEST(PrattParserRecursionDepthTest, SequentialScopesDoNotAccumulateDepth) { cel::ParserOptions options; options.max_recursion_depth = 2; diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index 86319a853..b7356528d 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -123,7 +123,7 @@ std::string ParserWorker::GetTokenText(const Token& tok) const { return ""; } -Token ParserWorker::NextSignificantToken() { +Token ParserWorker::NextSignificantToken(bool report_error) { if (is_recovery_limit_exceeded()) { return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; } @@ -132,7 +132,7 @@ Token ParserWorker::NextSignificantToken() { if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) { continue; } - if (tok.type == TokenType::kError) { + if (tok.type == TokenType::kError && report_error) { ReportError(tok, lexer_.GetError().message); if (is_recovery_limit_exceeded()) { return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 35e9ff52b..d6c1b385e 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -27,6 +27,7 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" +#include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_map.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" @@ -72,7 +73,7 @@ class ParserWorker { const cel::ParserOptions& options() const { return options_; } // Token stream management void InitTokenStream(); - Token NextSignificantToken(); + Token NextSignificantToken(bool report_error = true); Token NextToken(); bool Expect(TokenType type, absl::string_view msg = ""); std::string GetTokenText(const Token& tok) const; @@ -317,6 +318,8 @@ class PrattParserWorker : public ParserWorker { std::optional target, std::vector arguments); + int CountGroupingParentheses(); + AstFactoryInterface& ast_factory_; absl::flat_hash_map macro_calls_; }; @@ -652,9 +655,14 @@ PrattParserWorker::ParsePrimary() { ExprNode expr; TokenType tok_type = peek_token_.type; if (tok_type == TokenType::kLeftParen) { - NextToken(); + int grouping_paren_count = CountGroupingParentheses(); + for (int i = 0; i < grouping_paren_count; ++i) { + NextToken(); + } expr = ParseExpr(); - Expect(TokenType::kRightParen); + for (int i = 0; i < grouping_paren_count; ++i) { + Expect(TokenType::kRightParen); + } } else if (tok_type == TokenType::kNull) { Token tok = NextToken(); expr = ast_factory_.NewNullConst(NextId(tok)); @@ -1172,6 +1180,75 @@ void PrattParserWorker::RecordMacroCall( macro_calls_.insert({macro_id, std::move(call_expr)}); } +// Scans ahead in the token stream to detect contiguous grouping +// parentheses (e.g., `((((expr))))`). By determining the number of outermost +// parentheses that enclose the exact same expression and close contiguously, +// the parser unnests them in a single C++ stack frame, avoiding deep recursive +// descent. +template +int PrattParserWorker::CountGroupingParentheses() { + if (peek_token_.type != TokenType::kLeftParen) { + return 0; + } + + // Save lexer position to restore after scanning ahead. + const Lexer::Position saved_pos = lexer_.SavePosition(); + auto restore_lexer = absl::MakeCleanup( + [this, saved_pos] { lexer_.RestorePosition(saved_pos); }); + + int leading_open_parens = 1; + Token tok = this->NextSignificantToken(/*report_error=*/false); + while (tok.type == TokenType::kLeftParen) { + leading_open_parens++; + tok = this->NextSignificantToken(/*report_error=*/false); + } + if (leading_open_parens == 1) { + return 1; + } + + int open_parens = leading_open_parens; + int consecutive_leading_closed = 0; + + while (open_parens > 0) { + if (tok.type == TokenType::kEnd || tok.type == TokenType::kError) { + // Return 1 to ensure the parser consumes '(' and standard error handling + // catches incomplete expressions like `(ident`. + return 1; + } + + if (tok.type == TokenType::kLeftParen) { + // An inner parenthesis opens within the expression + // (e.g. `(x` in `((1 + (x) ))`). + open_parens++; + consecutive_leading_closed = 0; + } else if (tok.type == TokenType::kRightParen) { + if (leading_open_parens == open_parens) { + // All inner parentheses are balanced, so this ')' closes one of the + // initial leading '(' parentheses (e.g. trailing ')' in `(((expr)))`). + leading_open_parens--; + consecutive_leading_closed++; + } else { + // This ')' closes an inner nested parenthesis (e.g. `(1 + 2)` in + // `((1 + 2) * 3)`), not one of the outermost leading parentheses. + consecutive_leading_closed = 0; + } + open_parens--; + } else { + // Non-parenthesis token (identifier, operator, literal, etc.). Any + // preceding ')' did not close the entire expression, so reset the + // contiguous outer closing count. + consecutive_leading_closed = 0; + } + + if (open_parens > 0) { + tok = this->NextSignificantToken(/*report_error=*/false); + } + } + + // Return at least 1 to make sure we catch unclosed expressions like `(ident`. + return std::max(1, consecutive_leading_closed); +} + } // namespace cel::parser_internal #endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ diff --git a/parser/parser_test.cc b/parser/parser_test.cc index a7a2eb442..f7870b943 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1836,8 +1836,9 @@ TEST_P(ExpressionImplTest, RecursionDepthLongArgList) { EXPECT_THAT(Parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]", "", options_), IsOk()); } -TEST(ExpressionTest, RecursionDepthExceeded) { +TEST(ExpressionTest, RecursionDepthExceeded_AntlrOnly) { ParserOptions options; + options.enable_pratt_parser = false; // AST visitor will recurse a variable amount depending on the terms used in // the expression. This check occurs in the business logic converting the raw // Antlr parse tree into an Expr. There is a separate check (via a custom @@ -1877,10 +1878,9 @@ TEST_P(ExpressionImplTest, DisableStandardMacros) { << adorned_string; } -TEST(ExpressionTest, RecursionDepthIgnoresParentheses) { - ParserOptions options; - options.max_recursion_depth = 6; - auto result = Parse("(((1 + 2 + 3 + 4 + (5 + 6))))", "", options); +TEST_P(ExpressionImplTest, RecursionDepthIgnoresParentheses) { + options_.max_recursion_depth = options_.enable_pratt_parser ? 2 : 6; + auto result = Parse("(((1 + 2 + 3 + 4 + (5 + 6))))", "", options_); EXPECT_THAT(result, IsOk()); } From 02ddc546204836ab4bc668f154856e65dee3ca9b Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Sat, 15 Aug 2026 08:36:25 -0700 Subject: [PATCH 179/350] [Pratt Parser] Optimize call stack depth via in-place AST mutation and lean dispatch - In-place AST mutation in binary/ternary operations: - Updated some Parse* functions to accept ExprNode& lhs and mutate lhs in place instead of returning ExprNode by value. - Eliminated some intermediate named temporary rhs stack variables. - Lean ParseUnary and ParsePrimary dispatch: - Extracted prefix operator parsing (!, -) into ParseUnaryOps(), making ParseUnary() a fast single branch for standard non-prefix expressions. PiperOrigin-RevId: 965211950 --- parser/internal/pratt_parser_worker.h | 172 +++++++++++++------------- 1 file changed, 89 insertions(+), 83 deletions(-) diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index d6c1b385e..bbd719e34 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -223,7 +223,7 @@ class PrattParserWorker : public ParserWorker { // Parses ternary conditional expressions (`condition ? true_expr : // false_expr`). - ExprNode ParseTernary(ExprNode lhs); + void ParseTernary(ExprNode& lhs); // Helper method for parsing a contiguous chain of same-precedence logical // operators (`&&` or `||`) iteratively into a list of terms and operator IDs. @@ -234,7 +234,7 @@ class PrattParserWorker : public ParserWorker { // Example (`a && b && c && d`): Iteratively collects terms `[a, b, c, d]` and // builds `((a && b) && c) && d` without ascending/descending C++ stack // frames for each term. - ExprNode ParseBalancedLogicalChain(ExprNode lhs, const BinaryOpInfo& op_info); + void ParseBalancedLogicalChain(ExprNode& lhs, const BinaryOpInfo& op_info); // Parses prefix unary operators (`!`, `-`) and trailing postfix // member/indexing operations (`.field`, `[index]`, `.method(args)`). First @@ -252,7 +252,7 @@ class PrattParserWorker : public ParserWorker { // `Type{field: val}`). // // Processes continuous postfix operation chains iteratively. - ExprNode ParseSelectorChainTail(ExprNode lhs); + void ParseSelectorChainTail(ExprNode& lhs); // Parses prefix unary operators (logical NOT `!` and negation `-`). If a // numeric literal immediately follows `-`, folds it directly into a negative @@ -266,6 +266,9 @@ class PrattParserWorker : public ParserWorker { // wrapping `has(x.y)`. ExprNode ParseUnary(); + // Parses unary operators (`!`, `-`). + ExprNode ParseUnaryOps(); + // Parses unary operator chains (`!`, `-`). ExprNode ParseUnaryOpsChain(Token first_op); @@ -284,7 +287,7 @@ class PrattParserWorker : public ParserWorker { // Example (`(a + b)`): Consumes `(`, recurses to `ParseExpr()`, and expects // `)`. Example (`has(x.y)`): Consumes `has`, parses arguments `(x.y)`, and // expands the `has` macro. - ABSL_ATTRIBUTE_ALWAYS_INLINE inline ExprNode ParsePrimary(); + ExprNode ParsePrimary(); ExprNode ParseList(); ExprNode ParseMap(); @@ -297,8 +300,8 @@ class PrattParserWorker : public ParserWorker { ExprNode ParseNegativeDoubleLiteral(int64_t node_id); ExprNode ParseStringLiteral(); ExprNode ParseBytesLiteral(); - ExprNode BuildBinaryCall(int64_t op_id, absl::string_view op_name, - ExprNode lhs, ExprNode rhs); + void BuildBinaryCall(int64_t op_id, absl::string_view op_name, ExprNode& lhs, + ExprNode rhs); ExprNode ParseIdentOrCall(); std::string NormalizeIdent(const Token& tok, bool allow_quoted); std::optional ExtractStructName(const ExprNode& expr); @@ -353,12 +356,12 @@ ExprNode PrattParserWorker::ParseExpr() { } template -ExprNode PrattParserWorker::ParseTernary(ExprNode lhs) { +void PrattParserWorker::ParseTernary(ExprNode& lhs) { NextToken(); int64_t op_id = NextId(); ExprNode true_expr = ParseBinaryAndTernary(1); if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { - return lhs; + return; } ExprNode false_expr = ParseBinaryAndTernary(0); std::vector args; @@ -366,21 +369,20 @@ ExprNode PrattParserWorker::ParseTernary(ExprNode lhs) { args.push_back(std::move(lhs)); args.push_back(std::move(true_expr)); args.push_back(std::move(false_expr)); - return ast_factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(args)); + lhs = ast_factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(args)); } const BinaryOpInfo& GetBinaryOpInfo(TokenType type); template -ExprNode PrattParserWorker::BuildBinaryCall(int64_t op_id, - absl::string_view op_name, - ExprNode lhs, - ExprNode rhs) { +void PrattParserWorker::BuildBinaryCall(int64_t op_id, + absl::string_view op_name, + ExprNode& lhs, ExprNode rhs) { std::vector args; args.reserve(2); args.push_back(std::move(lhs)); args.push_back(std::move(rhs)); - return ast_factory_.NewCall(op_id, std::string(op_name), std::move(args)); + lhs = ast_factory_.NewCall(op_id, std::string(op_name), std::move(args)); } // Parses binary operator expressions and ternary conditional expressions @@ -391,7 +393,7 @@ ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { while (true) { TokenType tok = peek_token_.type; if (tok == TokenType::kQuestion && min_prec <= 0) { - lhs = ParseTernary(std::move(lhs)); + ParseTernary(lhs); continue; } @@ -399,14 +401,14 @@ ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { if (op_info.precedence < min_prec || op_info.precedence == 0) break; if (op_info.is_logical) { - lhs = ParseBalancedLogicalChain(std::move(lhs), op_info); + ParseBalancedLogicalChain(lhs, op_info); continue; } Token op_tok = NextToken(); int64_t op_id = NextId(op_tok); - ExprNode rhs = ParseBinaryAndTernary(op_info.precedence + 1); - lhs = BuildBinaryCall(op_id, op_info.name, std::move(lhs), std::move(rhs)); + BuildBinaryCall(op_id, op_info.name, lhs, + ParseBinaryAndTernary(op_info.precedence + 1)); } return lhs; } @@ -414,8 +416,8 @@ ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { // Parses continuous chains of logical operators (`&&`, `||`) iteratively // (e.g., `a && b && c`) and constructs a balanced or variadic AST. template -ExprNode PrattParserWorker::ParseBalancedLogicalChain( - ExprNode lhs, const BinaryOpInfo& op_info) { +void PrattParserWorker::ParseBalancedLogicalChain( + ExprNode& lhs, const BinaryOpInfo& op_info) { std::vector terms; std::vector ops; terms.push_back(std::move(lhs)); @@ -425,8 +427,8 @@ ExprNode PrattParserWorker::ParseBalancedLogicalChain( ops.push_back(NextId(op_tok)); terms.push_back(std::move(rhs)); } - return BalanceLogical(op_info.name, std::move(terms), std::move(ops), - options_.enable_variadic_logical_operators); + lhs = BalanceLogical(op_info.name, std::move(terms), std::move(ops), + options_.enable_variadic_logical_operators); } template @@ -435,7 +437,7 @@ ExprNode PrattParserWorker::ParseSelectorChain() { TokenType tok = peek_token_.type; if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || tok == TokenType::kLeftBrace) { - return ParseSelectorChainTail(std::move(lhs)); + ParseSelectorChainTail(lhs); } return lhs; } @@ -443,7 +445,7 @@ ExprNode PrattParserWorker::ParseSelectorChain() { // Parses prefix and postfix member/indexing operations iteratively // (e.g., `!a.b[0].c(x)`). template -ExprNode PrattParserWorker::ParseSelectorChainTail(ExprNode lhs) { +void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { while (true) { TokenType tok = peek_token_.type; if (tok == TokenType::kDot) { @@ -463,7 +465,7 @@ ExprNode PrattParserWorker::ParseSelectorChainTail(ExprNode lhs) { ReportError(id_tok, "expected identifier after '.'"); } SynchronizeOnDelimiter(); - return lhs; + return; } bool is_member_call = peek_token_.type == TokenType::kLeftParen; std::string id_text = @@ -475,7 +477,8 @@ ExprNode PrattParserWorker::ParseSelectorChainTail(ExprNode lhs) { args.push_back(std::move(lhs)); args.push_back( ast_factory_.NewStringConst(NextId(id_tok), std::move(id_text))); - lhs = ast_factory_.NewCall(op_id, "_?._", std::move(args)); + lhs = ast_factory_.NewCall(op_id, CelOperator::OPT_SELECT, + std::move(args)); } else if (peek_token_.type == TokenType::kLeftParen) { Token lparen = NextToken(); int64_t call_id = NextId(lparen); @@ -508,8 +511,9 @@ ExprNode PrattParserWorker::ParseSelectorChainTail(ExprNode lhs) { args.reserve(2); args.push_back(std::move(lhs)); args.push_back(std::move(index)); - lhs = ast_factory_.NewCall(op_id, optional ? "_[?_]" : CelOperator::INDEX, - std::move(args)); + lhs = ast_factory_.NewCall( + op_id, optional ? CelOperator::OPT_INDEX : CelOperator::INDEX, + std::move(args)); } else if (tok == TokenType::kLeftBrace) { int32_t struct_pos = GetLeftmostPosition(lhs); if (auto struct_name = ExtractStructName(lhs); struct_name.has_value()) { @@ -521,7 +525,6 @@ ExprNode PrattParserWorker::ParseSelectorChainTail(ExprNode lhs) { break; } } - return lhs; } template @@ -574,10 +577,14 @@ ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { template ExprNode PrattParserWorker::ParseUnary() { TokenType tok = peek_token_.type; - if (tok != TokenType::kExclamation && tok != TokenType::kMinus) { - return ParsePrimary(); + if (tok == TokenType::kExclamation || tok == TokenType::kMinus) { + return ParseUnaryOps(); } + return ParsePrimary(); +} +template +ExprNode PrattParserWorker::ParseUnaryOps() { Token op = NextToken(); TokenType op_type = op.type; if (peek_token_.type == TokenType::kExclamation || @@ -650,57 +657,59 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { // (`[...]`, `{...}`), and identifiers/global function calls (`foo`, // `has(x.y)`). template -ABSL_ATTRIBUTE_ALWAYS_INLINE inline ExprNode -PrattParserWorker::ParsePrimary() { - ExprNode expr; - TokenType tok_type = peek_token_.type; - if (tok_type == TokenType::kLeftParen) { - int grouping_paren_count = CountGroupingParentheses(); - for (int i = 0; i < grouping_paren_count; ++i) { - NextToken(); +ExprNode PrattParserWorker::ParsePrimary() { + switch (peek_token_.type) { + case TokenType::kLeftParen: { + int grouping_paren_count = CountGroupingParentheses(); + for (int i = 0; i < grouping_paren_count; ++i) { + NextToken(); + } + ExprNode expr = ParseExpr(); + for (int i = 0; i < grouping_paren_count; ++i) { + Expect(TokenType::kRightParen); + } + return expr; } - expr = ParseExpr(); - for (int i = 0; i < grouping_paren_count; ++i) { - Expect(TokenType::kRightParen); + case TokenType::kNull: + return ast_factory_.NewNullConst(NextId(NextToken())); + case TokenType::kTrue: + case TokenType::kFalse: { + Token tok = NextToken(); + return ast_factory_.NewBoolConst(NextId(tok), + tok.type == TokenType::kTrue); } - } else if (tok_type == TokenType::kNull) { - Token tok = NextToken(); - expr = ast_factory_.NewNullConst(NextId(tok)); - } else if (tok_type == TokenType::kTrue || tok_type == TokenType::kFalse) { - Token tok = NextToken(); - expr = ast_factory_.NewBoolConst(NextId(tok), tok_type == TokenType::kTrue); - } else if (tok_type == TokenType::kInt) { - expr = ParseIntLiteral(); - } else if (tok_type == TokenType::kUint) { - expr = ParseUintLiteral(); - } else if (tok_type == TokenType::kFloat) { - expr = ParseDoubleLiteral(); - } else if (tok_type == TokenType::kString) { - expr = ParseStringLiteral(); - } else if (tok_type == TokenType::kBytes) { - expr = ParseBytesLiteral(); - } else if (tok_type == TokenType::kLeftBracket) { - expr = ParseList(); - } else if (tok_type == TokenType::kLeftBrace) { - expr = ParseMap(); - } else if (tok_type == TokenType::kDot || tok_type == TokenType::kIdent || - tok_type == TokenType::kReservedWord) { - expr = ParseIdentOrCall(); - } else { - Token bad_tok = NextToken(); - if (bad_tok.type != TokenType::kError) { - if (bad_tok.type == TokenType::kEnd) { - ReportError( - bad_tok, - "Syntax error: mismatched input '' expecting expression"); - } else { - ReportError(bad_tok, "unexpected token"); + case TokenType::kInt: + return ParseIntLiteral(); + case TokenType::kUint: + return ParseUintLiteral(); + case TokenType::kFloat: + return ParseDoubleLiteral(); + case TokenType::kString: + return ParseStringLiteral(); + case TokenType::kBytes: + return ParseBytesLiteral(); + case TokenType::kLeftBracket: + return ParseList(); + case TokenType::kLeftBrace: + return ParseMap(); + case TokenType::kDot: + case TokenType::kIdent: + case TokenType::kReservedWord: + return ParseIdentOrCall(); + default: { + Token bad_tok = NextToken(); + if (bad_tok.type != TokenType::kError) { + if (bad_tok.type == TokenType::kEnd) { + ReportError( + bad_tok, + "Syntax error: mismatched input '' expecting expression"); + } else { + ReportError(bad_tok, "unexpected token"); + } } + return ast_factory_.NewUnspecified(NextId(bad_tok)); } - expr = ast_factory_.NewUnspecified(NextId(bad_tok)); } - - return expr; } // Parses list creation literals (e.g., `[1, 2, ?3]`). @@ -719,8 +728,7 @@ ExprNode PrattParserWorker::ParseList() { ReportError(q, "unsupported syntax '?'"); } } - ExprNode elem = ParseExpr(); - builder.Add(std::move(elem), optional); + builder.Add(ParseExpr(), optional); if (peek_token_.type == TokenType::kComma) { NextToken(); } else { @@ -757,8 +765,7 @@ ExprNode PrattParserWorker::ParseMap() { break; } int64_t entry_id = NextId(colon); - ExprNode val = ParseExpr(); - builder.Add(entry_id, std::move(key), std::move(val), optional); + builder.Add(entry_id, std::move(key), ParseExpr(), optional); if (peek_token_.type == TokenType::kComma) { NextToken(); } else { @@ -801,8 +808,7 @@ ExprNode PrattParserWorker::ParseStruct( break; } int64_t field_id = NextId(colon); - ExprNode val = ParseExpr(); - builder.Add(field_id, std::move(field_name), std::move(val), optional); + builder.Add(field_id, std::move(field_name), ParseExpr(), optional); if (peek_token_.type == TokenType::kComma) { NextToken(); } else { From e6485ab94a6a4f1a9aead2b53e64ce2389917c1f Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Sat, 15 Aug 2026 10:12:30 -0700 Subject: [PATCH 180/350] [Pratt Parser] Add BUILD target that excludes the ANTLR4 implementation Binary size comparison (`--config=release`): - Shared library (`libpratt_parser.so` vs `libparser.so`): 94.8 KB vs 454.6 KB (~360 KB / 79.15% reduction) - Static library (`libpratt_parser.a` vs `libparser.a`): 1.40 MB vs 4.34 MB (~2.94 MB / 67.73% reduction) - Static library, stripped: 127 KB vs 590.3 KB (~460 KB / 78.5% reduction) - `.text` section (executable code): 37 KB vs 212 KB (~175 KB / 82.57% reduction) PiperOrigin-RevId: 965233723 --- parser/BUILD | 82 +- parser/internal/BUILD | 67 + parser/internal/antlr_parser.cc | 1849 ++++++++++++++++++++++++ parser/internal/antlr_parser.h | 106 ++ parser/internal/antlr_parser_test.cc | 76 + parser/parser.cc | 1979 +------------------------- parser/parser_test.cc | 53 +- 7 files changed, 2229 insertions(+), 1983 deletions(-) create mode 100644 parser/internal/antlr_parser.cc create mode 100644 parser/internal/antlr_parser.h create mode 100644 parser/internal/antlr_parser_test.cc diff --git a/parser/BUILD b/parser/BUILD index 383d82e50..f29765835 100644 --- a/parser/BUILD +++ b/parser/BUILD @@ -27,49 +27,50 @@ cc_library( hdrs = [ "parser.h", ], - copts = [ - "-fexceptions", + deps = [ + ":macro", + ":macro_registry", + ":options", + ":parser_interface", + ":source_factory", + "//common:ast", + "//common:source", + "//common/ast:expr_proto", + "//common/ast:source_info_proto", + "//internal:status_macros", + "//parser/internal:antlr_parser", + "//parser/internal:pratt_parser", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + ], +) + +cc_library( + name = "pratt_parser", + srcs = [ + "parser.cc", + ], + hdrs = [ + "parser.h", ], defines = [ - "ANTLR4CPP_STATIC", + "EXCLUDE_CEL_ANTLR_PARSER", ], deps = [ ":macro", - ":macro_expr_factory", ":macro_registry", ":options", ":parser_interface", ":source_factory", "//common:ast", - "//common:constant", - "//common:expr", - "//common:expr_factory", - "//common:operators", "//common:source", "//common/ast:expr_proto", "//common/ast:source_info_proto", - "//internal:lexis", "//internal:status_macros", - "//internal:strings", - "//parser/internal:cel_cc_parser", "//parser/internal:pratt_parser", - "@antlr4-cpp-runtime", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/cleanup", - "@com_google_absl//absl/container:btree", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/functional:overload", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/log:check", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", - "@com_google_absl//absl/types:variant", + "@com_google_absl//absl/strings:string_view", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", ], ) @@ -202,6 +203,33 @@ cc_test( ], ) +cc_test( + name = "pratt_parser_test", + srcs = ["parser_test.cc"], + defines = [ + "EXCLUDE_CEL_ANTLR_PARSER", + ], + deps = [ + ":macro", + ":options", + ":parser_interface", + ":pratt_parser", + ":source_factory", + "//common:constant", + "//common:expr", + "//common:expr_printer", + "//common:source", + "//internal:testing", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:optional", + "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + ], +) + cc_test( name = "parser_benchmarks", srcs = ["parser_benchmarks.cc"], diff --git a/parser/internal/BUILD b/parser/internal/BUILD index cdc73288a..cebb5c329 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -62,6 +62,73 @@ antlr_cc_library( package = "cel_parser_internal", ) +cc_library( + name = "antlr_parser", + srcs = ["antlr_parser.cc"], + hdrs = ["antlr_parser.h"], + copts = [ + "-fexceptions", + ], + defines = [ + "ANTLR4CPP_STATIC", + ], + features = ["-use_header_modules"], # Incompatible with -fexceptions + deps = [ + ":cel_cc_parser", + "//common:ast", + "//common:constant", + "//common:expr", + "//common:expr_factory", + "//common:operators", + "//common:source", + "//internal:lexis", + "//internal:status_macros", + "//internal:strings", + "//parser:macro", + "//parser:macro_expr_factory", + "//parser:macro_registry", + "//parser:options", + "//parser:parser_interface", + "//parser:source_factory", + "@antlr4-cpp-runtime", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/functional:overload", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + "@com_google_absl//absl/types:variant", + ], +) + +cc_test( + name = "antlr_parser_test", + srcs = ["antlr_parser_test.cc"], + deps = [ + ":antlr_parser", + "//common:ast", + "//common:source", + "//internal:status_macros", + "//internal:testing", + "//parser:options", + "//parser:parser_interface", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + ], +) + cc_library( name = "lexer", srcs = ["lexer.cc"], diff --git a/parser/internal/antlr_parser.cc b/parser/internal/antlr_parser.cc new file mode 100644 index 000000000..6116d05a3 --- /dev/null +++ b/parser/internal/antlr_parser.cc @@ -0,0 +1,1849 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/antlr_parser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/macros.h" +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/btree_map.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/functional/overload.h" +#include "absl/log/absl_check.h" +#include "absl/log/check.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "absl/types/variant.h" +#include "antlr4-runtime.h" +#include "common/ast.h" +#include "common/constant.h" +#include "common/expr.h" +#include "common/expr_factory.h" +#include "common/operators.h" +#include "common/source.h" +#include "internal/lexis.h" +#include "internal/status_macros.h" +#include "internal/strings.h" +#pragma push_macro("IN") +#undef IN +#include "parser/internal/CelBaseVisitor.h" +#include "parser/internal/CelLexer.h" +#include "parser/internal/CelParser.h" +#pragma pop_macro("IN") +#include "parser/macro.h" +#include "parser/macro_expr_factory.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" +#include "parser/source_factory.h" + +namespace cel { + +namespace { + +constexpr const char kHiddenAccumulatorVariableName[] = "@result"; + +std::any ExprPtrToAny(std::unique_ptr&& expr) { + return std::make_any(expr.release()); +} + +std::any ExprToAny(Expr&& expr) { + return ExprPtrToAny(std::make_unique(std::move(expr))); +} + +std::unique_ptr ExprPtrFromAny(std::any&& any) { + return absl::WrapUnique(std::any_cast(std::move(any))); +} + +Expr ExprFromAny(std::any&& any) { + auto expr = ExprPtrFromAny(std::move(any)); + return std::move(*expr); +} + +struct ParserError { + std::string message; + SourceRange range; +}; + +std::string DisplayParserError(const cel::Source& source, + SourceLocation location, + absl::string_view message) { + return absl::StrCat(absl::StrFormat("ERROR: %s:%zu:%zu: %s", + source.description(), location.line, + // add one to the 0-based column + location.column + 1, message), + source.DisplayErrorLocation(location)); +} + +int32_t PositiveOrMax(int32_t value) { + return value >= 0 ? value : std::numeric_limits::max(); +} + +SourceRange SourceRangeFromToken(const antlr4::Token* token) { + SourceRange range; + if (token != nullptr) { + if (auto start = token->getStartIndex(); start != INVALID_INDEX) { + range.begin = static_cast(start); + } + if (auto end = token->getStopIndex(); end != INVALID_INDEX) { + range.end = static_cast(end + 1); + } + } + return range; +} + +SourceRange SourceRangeFromParserRuleContext( + const antlr4::ParserRuleContext* context) { + SourceRange range; + if (context != nullptr) { + if (auto start = context->getStart() != nullptr + ? context->getStart()->getStartIndex() + : INVALID_INDEX; + start != INVALID_INDEX) { + range.begin = static_cast(start); + } + if (auto end = context->getStop() != nullptr + ? context->getStop()->getStopIndex() + : INVALID_INDEX; + end != INVALID_INDEX) { + range.end = static_cast(end + 1); + } + } + return range; +} + +} // namespace + +class ParserMacroExprFactory final : public MacroExprFactory { + public: + explicit ParserMacroExprFactory(const cel::Source& source, + int expression_node_limit) + : source_(source), expression_node_limit_(expression_node_limit) {} + + void BeginMacro(SourceRange macro_position) { + macro_position_ = macro_position; + } + + void EndMacro() { macro_position_ = SourceRange{}; } + + Expr ReportError(absl::string_view message) override { + return ReportError(macro_position_, message); + } + + Expr ReportError(int64_t expr_id, absl::string_view message) { + return ReportError(GetSourceRange(expr_id), message); + } + + Expr ReportError(SourceRange range, absl::string_view message) { + ++error_count_; + if (errors_.size() <= 100) { + errors_.push_back(ParserError{std::string(message), range}); + } + return NewUnspecified(NextId(range)); + } + + Expr ReportErrorAt(const Expr& expr, absl::string_view message) override { + return ReportError(GetSourceRange(expr.id()), message); + } + + SourceRange GetSourceRange(int64_t id) const { + if (auto it = positions_.find(id); it != positions_.end()) { + return it->second; + } + return SourceRange{}; + } + + int64_t NextId(const SourceRange& range) { + auto id = expr_id_++; + if (id > expression_node_limit_ && !node_limit_exceeded_) { + node_limit_exceeded_ = true; + ReportError(range, "expression node limit exceeded"); + } + if (range.begin != -1 || range.end != -1) { + positions_.insert(std::pair{id, range}); + } + return id; + } + + bool is_node_limit_exceeded() const { return node_limit_exceeded_; } + + bool HasErrors() const { return error_count_ != 0; } + + std::vector CollectIssues() { + // Errors are collected as they are encountered, not by their location + // within the source. To have a more stable error message as implementation + // details change, we sort the collected errors by their source location + // first. + std::stable_sort( + errors_.begin(), errors_.end(), + [](const ParserError& lhs, const ParserError& rhs) -> bool { + auto lhs_begin = PositiveOrMax(lhs.range.begin); + auto lhs_end = PositiveOrMax(lhs.range.end); + auto rhs_begin = PositiveOrMax(rhs.range.begin); + auto rhs_end = PositiveOrMax(rhs.range.end); + return lhs_begin < rhs_begin || + (lhs_begin == rhs_begin && lhs_end < rhs_end); + }); + // Build the summary error message using the sorted errors. + bool errors_truncated = error_count_ > 100; + std::vector issues; + issues.reserve( + errors_.size() + + errors_truncated); // Reserve space for the transform and an + // additional element when truncation occurs. + std::transform( + errors_.begin(), errors_.end(), std::back_inserter(issues), + [this](const ParserError& error) { + auto location = + source_.GetLocation(error.range.begin).value_or(SourceLocation{}); + return cel::ParseIssue(location, error.message); + }); + if (errors_truncated) { + issues.push_back(cel::ParseIssue( + absl::StrCat(error_count_ - 100, " more errors were truncated."))); + } + return issues; + } + + void AddMacroCall(int64_t macro_id, absl::string_view function, + absl::optional target, std::vector arguments) { + macro_calls_.insert( + {macro_id, target.has_value() + ? NewMemberCall(0, function, std::move(*target), + std::move(arguments)) + : NewCall(0, function, std::move(arguments))}); + } + + Expr BuildMacroCallArg(const Expr& expr) { + if (auto it = macro_calls_.find(expr.id()); it != macro_calls_.end()) { + return NewUnspecified(expr.id()); + } + return absl::visit( + absl::Overload( + [this, &expr](const UnspecifiedExpr&) -> Expr { + return NewUnspecified(expr.id()); + }, + [this, &expr](const Constant& const_expr) -> Expr { + return NewConst(expr.id(), const_expr); + }, + [this, &expr](const IdentExpr& ident_expr) -> Expr { + return NewIdent(expr.id(), ident_expr.name()); + }, + [this, &expr](const SelectExpr& select_expr) -> Expr { + return select_expr.test_only() + ? NewPresenceTest( + expr.id(), + BuildMacroCallArg(select_expr.operand()), + select_expr.field()) + : NewSelect(expr.id(), + BuildMacroCallArg(select_expr.operand()), + select_expr.field()); + }, + [this, &expr](const CallExpr& call_expr) -> Expr { + std::vector macro_arguments; + macro_arguments.reserve(call_expr.args().size()); + for (const auto& argument : call_expr.args()) { + macro_arguments.push_back(BuildMacroCallArg(argument)); + } + absl::optional macro_target; + if (call_expr.has_target()) { + macro_target = BuildMacroCallArg(call_expr.target()); + } + return macro_target.has_value() + ? NewMemberCall(expr.id(), call_expr.function(), + std::move(*macro_target), + std::move(macro_arguments)) + : NewCall(expr.id(), call_expr.function(), + std::move(macro_arguments)); + }, + [this, &expr](const ListExpr& list_expr) -> Expr { + std::vector macro_elements; + macro_elements.reserve(list_expr.elements().size()); + for (const auto& element : list_expr.elements()) { + auto& cloned_element = macro_elements.emplace_back(); + if (element.has_expr()) { + cloned_element.set_expr(BuildMacroCallArg(element.expr())); + } + cloned_element.set_optional(element.optional()); + } + return NewList(expr.id(), std::move(macro_elements)); + }, + [this, &expr](const StructExpr& struct_expr) -> Expr { + std::vector macro_fields; + macro_fields.reserve(struct_expr.fields().size()); + for (const auto& field : struct_expr.fields()) { + auto& macro_field = macro_fields.emplace_back(); + macro_field.set_id(field.id()); + macro_field.set_name(field.name()); + macro_field.set_value(BuildMacroCallArg(field.value())); + macro_field.set_optional(field.optional()); + } + return NewStruct(expr.id(), struct_expr.name(), + std::move(macro_fields)); + }, + [this, &expr](const MapExpr& map_expr) -> Expr { + std::vector macro_entries; + macro_entries.reserve(map_expr.entries().size()); + for (const auto& entry : map_expr.entries()) { + auto& macro_entry = macro_entries.emplace_back(); + macro_entry.set_id(entry.id()); + macro_entry.set_key(BuildMacroCallArg(entry.key())); + macro_entry.set_value(BuildMacroCallArg(entry.value())); + macro_entry.set_optional(entry.optional()); + } + return NewMap(expr.id(), std::move(macro_entries)); + }, + [this, &expr](const ComprehensionExpr& comprehension_expr) -> Expr { + return NewComprehension( + expr.id(), comprehension_expr.iter_var(), + BuildMacroCallArg(comprehension_expr.iter_range()), + comprehension_expr.accu_var(), + BuildMacroCallArg(comprehension_expr.accu_init()), + BuildMacroCallArg(comprehension_expr.loop_condition()), + BuildMacroCallArg(comprehension_expr.loop_step()), + BuildMacroCallArg(comprehension_expr.result())); + }), + expr.kind()); + } + + using ExprFactory::NewBoolConst; + using ExprFactory::NewBytesConst; + using ExprFactory::NewCall; + using ExprFactory::NewComprehension; + using ExprFactory::NewConst; + using ExprFactory::NewDoubleConst; + using ExprFactory::NewIdent; + using ExprFactory::NewIntConst; + using ExprFactory::NewList; + using ExprFactory::NewListElement; + using ExprFactory::NewMap; + using ExprFactory::NewMapEntry; + using ExprFactory::NewMemberCall; + using ExprFactory::NewNullConst; + using ExprFactory::NewPresenceTest; + using ExprFactory::NewSelect; + using ExprFactory::NewStringConst; + using ExprFactory::NewStruct; + using ExprFactory::NewStructField; + using ExprFactory::NewUintConst; + using ExprFactory::NewUnspecified; + + const absl::btree_map& positions() const { + return positions_; + } + + const absl::flat_hash_map& macro_calls() const { + return macro_calls_; + } + + absl::flat_hash_map release_macro_calls() { + using std::swap; + absl::flat_hash_map result; + swap(result, macro_calls_); + return result; + } + + void EraseId(ExprId id) { + positions_.erase(id); + if (expr_id_ == id + 1) { + --expr_id_; + } + } + + protected: + int64_t NextId() override { return NextId(macro_position_); } + + int64_t CopyId(int64_t id) override { + if (id == 0) { + return 0; + } + return NextId(GetSourceRange(id)); + } + + private: + int64_t expr_id_ = 1; + absl::btree_map positions_; + absl::flat_hash_map macro_calls_; + std::vector errors_; + size_t error_count_ = 0; + const Source& source_; + int expression_node_limit_; + bool node_limit_exceeded_ = false; + SourceRange macro_position_; +}; + +} // namespace cel + +namespace cel::parser_internal { + +namespace { + +using ::antlr4::CharStream; +using ::antlr4::CommonTokenStream; +using ::antlr4::DefaultErrorStrategy; +using ::antlr4::ParseCancellationException; +using ::antlr4::Parser; +using ::antlr4::ParserRuleContext; +using ::antlr4::Token; +using ::antlr4::misc::IntervalSet; +using ::antlr4::tree::ErrorNode; +using ::antlr4::tree::ParseTreeListener; +using ::antlr4::tree::TerminalNode; +using ::cel_parser_internal::CelBaseVisitor; +using ::cel_parser_internal::CelLexer; +using ::cel_parser_internal::CelParser; +using ::google::api::expr::common::CelOperator; +using ::google::api::expr::common::ReverseLookupOperator; + +class CodePointStream final : public CharStream { + public: + CodePointStream(cel::SourceContentView buffer, absl::string_view source_name) + : buffer_(buffer), + source_name_(source_name), + size_(buffer_.size()), + index_(0) {} + + void consume() override { + if (ABSL_PREDICT_FALSE(index_ >= size_)) { + ABSL_ASSERT(LA(1) == IntStream::EOF); + throw antlr4::IllegalStateException("cannot consume EOF"); + } + index_++; + } + + size_t LA(ptrdiff_t i) override { + if (ABSL_PREDICT_FALSE(i == 0)) { + return 0; + } + auto p = static_cast(index_); + if (i < 0) { + i++; + if (p + i - 1 < 0) { + return IntStream::EOF; + } + } + if (p + i - 1 >= static_cast(size_)) { + return IntStream::EOF; + } + return buffer_.at(static_cast(p + i - 1)); + } + + ptrdiff_t mark() override { return -1; } + + void release(ptrdiff_t marker) override {} + + size_t index() override { return index_; } + + void seek(size_t index) override { index_ = std::min(index, size_); } + + size_t size() override { return size_; } + + std::string getSourceName() const override { + return source_name_.empty() ? IntStream::UNKNOWN_SOURCE_NAME + : std::string(source_name_); + } + + std::string getText(const antlr4::misc::Interval& interval) override { + if (ABSL_PREDICT_FALSE(interval.a < 0 || interval.b < 0)) { + return std::string(); + } + size_t start = static_cast(interval.a); + if (ABSL_PREDICT_FALSE(start >= size_)) { + return std::string(); + } + size_t stop = static_cast(interval.b); + if (ABSL_PREDICT_FALSE(stop >= size_)) { + stop = size_ - 1; + } + return buffer_.ToString(static_cast(start), + static_cast(stop) + 1); + } + + std::string toString() const override { return buffer_.ToString(); } + + private: + cel::SourceContentView const buffer_; + const absl::string_view source_name_; + const size_t size_; + size_t index_; +}; + +// Scoped helper for incrementing the parse recursion count. +// Increments on creation, decrements on destruction (stack unwind). +class ScopedIncrement final { + public: + explicit ScopedIncrement(int& recursion_depth) + : recursion_depth_(recursion_depth) { + ++recursion_depth_; + } + + ~ScopedIncrement() { --recursion_depth_; } + + private: + int& recursion_depth_; +}; + +// balancer performs tree balancing on operators whose arguments are of equal +// precedence. +// +// The purpose of the balancer is to ensure a compact serialization format for +// the logical &&, || operators which have a tendency to create long DAGs which +// are skewed in one direction. Since the operators are commutative re-ordering +// the terms *must not* affect the evaluation result. +// +// Based on code from //third_party/cel/go/parser/helper.go +class ExpressionBalancer final { + public: + ExpressionBalancer(cel::ParserMacroExprFactory& factory, std::string function, + Expr expr); + + // addTerm adds an operation identifier and term to the set of terms to be + // balanced. + void AddTerm(int64_t op, Expr term); + + // balance creates a balanced tree from the sub-terms and returns the final + // Expr value. + Expr Balance(bool enable_variadic = false); + + private: + // balancedTree recursively balances the terms provided to a commutative + // operator. + Expr BalancedTree(int lo, int hi); + + private: + cel::ParserMacroExprFactory& factory_; + std::string function_; + std::vector terms_; + std::vector ops_; +}; + +ExpressionBalancer::ExpressionBalancer(cel::ParserMacroExprFactory& factory, + std::string function, Expr expr) + : factory_(factory), function_(std::move(function)) { + terms_.push_back(std::move(expr)); +} + +void ExpressionBalancer::AddTerm(int64_t op, Expr term) { + terms_.push_back(std::move(term)); + ops_.push_back(op); +} + +Expr ExpressionBalancer::Balance(bool enable_variadic) { + if (terms_.size() == 1) { + return std::move(terms_[0]); + } + if (enable_variadic) { + return factory_.NewCall(ops_[0], function_, std::move(terms_)); + } + return BalancedTree(0, ops_.size() - 1); +} + +Expr ExpressionBalancer::BalancedTree(int lo, int hi) { + int mid = (lo + hi + 1) / 2; + + std::vector arguments; + arguments.reserve(2); + + if (mid == lo) { + arguments.push_back(std::move(terms_[mid])); + } else { + arguments.push_back(BalancedTree(lo, mid - 1)); + } + + if (mid == hi) { + arguments.push_back(std::move(terms_[mid + 1])); + } else { + arguments.push_back(BalancedTree(mid + 1, hi)); + } + return factory_.NewCall(ops_[mid], function_, std::move(arguments)); +} + +std::string FormatIssues(const cel::Source& source, + absl::Span issues) { + return absl::StrJoin( + issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { + absl::StrAppend( + out, DisplayParserError(source, issue.location(), issue.message())); + }); +} + +class ParserVisitor final : public CelBaseVisitor, + public antlr4::BaseErrorListener { + public: + ParserVisitor(const cel::Source& source, int max_recursion_depth, + int max_expression_node_count, + const cel::MacroRegistry& macro_registry, + bool add_macro_calls = false, + bool enable_optional_syntax = false, + bool enable_quoted_identifiers = false, + bool enable_variadic_logical_operators = false) + : source_(source), + factory_(source_, max_expression_node_count), + macro_registry_(macro_registry), + recursion_depth_(0), + max_recursion_depth_(max_recursion_depth), + add_macro_calls_(add_macro_calls), + enable_optional_syntax_(enable_optional_syntax), + enable_quoted_identifiers_(enable_quoted_identifiers), + enable_variadic_logical_operators_(enable_variadic_logical_operators) {} + + ~ParserVisitor() override = default; + + std::any visit(antlr4::tree::ParseTree* tree) override; + + std::any visitStart(CelParser::StartContext* ctx) override; + std::any visitExpr(CelParser::ExprContext* ctx) override; + std::any visitConditionalOr(CelParser::ConditionalOrContext* ctx) override; + std::any visitConditionalAnd(CelParser::ConditionalAndContext* ctx) override; + std::any visitRelation(CelParser::RelationContext* ctx) override; + std::any visitCalc(CelParser::CalcContext* ctx) override; + std::any visitUnary(CelParser::UnaryContext* ctx); + std::any visitLogicalNot(CelParser::LogicalNotContext* ctx) override; + std::any visitNegate(CelParser::NegateContext* ctx) override; + std::any visitSelect(CelParser::SelectContext* ctx) override; + std::any visitMemberCall(CelParser::MemberCallContext* ctx) override; + std::any visitIndex(CelParser::IndexContext* ctx) override; + std::any visitCreateMessage(CelParser::CreateMessageContext* ctx) override; + std::any visitFieldInitializerList( + CelParser::FieldInitializerListContext* ctx) override; + std::vector visitFields( + CelParser::FieldInitializerListContext* ctx); + std::any visitGlobalCall(CelParser::GlobalCallContext* ctx) override; + std::any visitIdent(CelParser::IdentContext* ctx) override; + std::any visitNested(CelParser::NestedContext* ctx) override; + std::any visitCreateList(CelParser::CreateListContext* ctx) override; + std::vector visitList(CelParser::ListInitContext* ctx); + std::vector visitList(CelParser::ExprListContext* ctx); + std::any visitCreateMap(CelParser::CreateMapContext* ctx) override; + std::any visitConstantLiteral( + CelParser::ConstantLiteralContext* ctx) override; + std::any visitPrimaryExpr(CelParser::PrimaryExprContext* ctx) override; + std::any visitMemberExpr(CelParser::MemberExprContext* ctx) override; + + std::any visitMapInitializerList( + CelParser::MapInitializerListContext* ctx) override; + std::vector visitEntries( + CelParser::MapInitializerListContext* ctx); + std::any visitInt(CelParser::IntContext* ctx) override; + std::any visitUint(CelParser::UintContext* ctx) override; + std::any visitDouble(CelParser::DoubleContext* ctx) override; + std::any visitString(CelParser::StringContext* ctx) override; + std::any visitBytes(CelParser::BytesContext* ctx) override; + std::any visitBoolTrue(CelParser::BoolTrueContext* ctx) override; + std::any visitBoolFalse(CelParser::BoolFalseContext* ctx) override; + std::any visitNull(CelParser::NullContext* ctx) override; + // Note: this is destructive and intended to be called after the parse is + // finished. + cel::SourceInfo GetSourceInfo(); + EnrichedSourceInfo enriched_source_info() const; + void syntaxError(antlr4::Recognizer* recognizer, + antlr4::Token* offending_symbol, size_t line, size_t col, + const std::string& msg, std::exception_ptr e) override; + bool HasErrored() const; + + std::vector CollectIssues(); + + private: + template + Expr GlobalCallOrMacro(int64_t expr_id, absl::string_view function, + Args&&... args) { + std::vector arguments; + arguments.reserve(sizeof...(Args)); + (arguments.push_back(std::forward(args)), ...); + return GlobalCallOrMacroImpl(expr_id, function, std::move(arguments)); + } + + Expr GlobalCallOrMacroImpl(int64_t expr_id, absl::string_view function, + std::vector args); + Expr ReceiverCallOrMacroImpl(int64_t expr_id, absl::string_view function, + Expr target, std::vector args); + std::string ExtractQualifiedName(antlr4::ParserRuleContext* ctx, + const Expr& e); + + std::string NormalizeIdentifier(CelParser::EscapeIdentContext* ctx); + // Attempt to unnest parse context. + // + // Walk the parse tree to the first complex term to reduce recursive depth in + // the visit* calls. + antlr4::tree::ParseTree* UnnestContext(antlr4::tree::ParseTree* tree); + + private: + const cel::Source& source_; + cel::ParserMacroExprFactory factory_; + const cel::MacroRegistry& macro_registry_; + int recursion_depth_; + const int max_recursion_depth_; + const bool add_macro_calls_; + const bool enable_optional_syntax_; + const bool enable_quoted_identifiers_; + const bool enable_variadic_logical_operators_; +}; + +template ::value>> +T* tree_as(antlr4::tree::ParseTree* tree) { + return dynamic_cast(tree); +} + +std::any ParserVisitor::visit(antlr4::tree::ParseTree* tree) { + ScopedIncrement inc(recursion_depth_); + if (recursion_depth_ > max_recursion_depth_) { + return ExprToAny(factory_.ReportError( + absl::StrFormat("Exceeded max recursion depth of %d when parsing.", + max_recursion_depth_))); + } + tree = UnnestContext(tree); + if (auto* ctx = tree_as(tree)) { + return visitStart(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitExpr(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitConditionalAnd(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitConditionalOr(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitRelation(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitCalc(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitLogicalNot(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitPrimaryExpr(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitMemberExpr(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitSelect(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitMemberCall(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitMapInitializerList(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitNegate(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitIndex(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitUnary(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitCreateList(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitCreateMessage(ctx); + } else if (auto* ctx = tree_as(tree)) { + return visitCreateMap(ctx); + } + + if (tree) { + return ExprToAny( + factory_.ReportError(SourceRangeFromParserRuleContext( + tree_as(tree)), + "unknown parsetree type")); + } + return ExprToAny(factory_.ReportError("<> parsetree")); +} + +std::any ParserVisitor::visitPrimaryExpr(CelParser::PrimaryExprContext* pctx) { + CelParser::PrimaryContext* primary = pctx->primary(); + if (auto* ctx = tree_as(primary)) { + return visitNested(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitIdent(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitGlobalCall(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitCreateList(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitCreateMap(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitCreateMessage(ctx); + } else if (auto* ctx = tree_as(primary)) { + return visitConstantLiteral(ctx); + } + if (factory_.HasErrors()) { + // ANTLR creates PrimaryContext rather than a derived class during certain + // error conditions. This is odd, but we ignore it as we already have errors + // that occurred. + return ExprToAny(factory_.NewUnspecified(factory_.NextId({}))); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(pctx), + "invalid primary expression")); +} + +std::any ParserVisitor::visitMemberExpr(CelParser::MemberExprContext* mctx) { + CelParser::MemberContext* member = mctx->member(); + if (auto* ctx = tree_as(member)) { + return visitPrimaryExpr(ctx); + } else if (auto* ctx = tree_as(member)) { + return visitSelect(ctx); + } else if (auto* ctx = tree_as(member)) { + return visitMemberCall(ctx); + } else if (auto* ctx = tree_as(member)) { + return visitIndex(ctx); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(mctx), + "unsupported simple expression")); +} + +std::any ParserVisitor::visitStart(CelParser::StartContext* ctx) { + return visit(ctx->expr()); +} + +antlr4::tree::ParseTree* ParserVisitor::UnnestContext( + antlr4::tree::ParseTree* tree) { + antlr4::tree::ParseTree* last = nullptr; + while (tree != last) { + last = tree; + + if (auto* ctx = tree_as(tree)) { + tree = ctx->expr(); + } + + if (auto* ctx = tree_as(tree)) { + if (ctx->op != nullptr) { + return ctx; + } + tree = ctx->e; + } + + if (auto* ctx = tree_as(tree)) { + if (!ctx->ops.empty()) { + return ctx; + } + tree = ctx->e; + } + + if (auto* ctx = tree_as(tree)) { + if (!ctx->ops.empty()) { + return ctx; + } + tree = ctx->e; + } + + if (auto* ctx = tree_as(tree)) { + if (ctx->calc() == nullptr) { + return ctx; + } + tree = ctx->calc(); + } + + if (auto* ctx = tree_as(tree)) { + if (ctx->unary() == nullptr) { + return ctx; + } + tree = ctx->unary(); + } + + if (auto* ctx = tree_as(tree)) { + tree = ctx->member(); + } + + if (auto* ctx = tree_as(tree)) { + if (auto* nested = tree_as(ctx->primary())) { + tree = nested->e; + } else { + return ctx; + } + } + } + + return tree; +} + +std::any ParserVisitor::visitExpr(CelParser::ExprContext* ctx) { + auto result = ExprFromAny(visit(ctx->e)); + if (!ctx->op) { + return ExprToAny(std::move(result)); + } + std::vector arguments; + arguments.reserve(3); + arguments.push_back(std::move(result)); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + arguments.push_back(ExprFromAny(visit(ctx->e1))); + arguments.push_back(ExprFromAny(visit(ctx->e2))); + + return ExprToAny( + factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(arguments))); +} + +std::any ParserVisitor::visitConditionalOr( + CelParser::ConditionalOrContext* ctx) { + auto result = ExprFromAny(visit(ctx->e)); + if (ctx->ops.empty()) { + return ExprToAny(std::move(result)); + } + ExpressionBalancer b(factory_, CelOperator::LOGICAL_OR, std::move(result)); + for (size_t i = 0; i < ctx->ops.size(); ++i) { + auto op = ctx->ops[i]; + if (i >= ctx->e1.size()) { + return ExprToAny( + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unexpected character, wanted '||'")); + } + auto next = ExprFromAny(visit(ctx->e1[i])); + int64_t op_id = factory_.NextId(SourceRangeFromToken(op)); + b.AddTerm(op_id, std::move(next)); + } + return ExprToAny(b.Balance(enable_variadic_logical_operators_)); +} + +std::any ParserVisitor::visitConditionalAnd( + CelParser::ConditionalAndContext* ctx) { + auto result = ExprFromAny(visit(ctx->e)); + if (ctx->ops.empty()) { + return ExprToAny(std::move(result)); + } + ExpressionBalancer b(factory_, CelOperator::LOGICAL_AND, std::move(result)); + for (size_t i = 0; i < ctx->ops.size(); ++i) { + auto op = ctx->ops[i]; + if (i >= ctx->e1.size()) { + return ExprToAny( + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unexpected character, wanted '&&'")); + } + auto next = ExprFromAny(visit(ctx->e1[i])); + int64_t op_id = factory_.NextId(SourceRangeFromToken(op)); + b.AddTerm(op_id, std::move(next)); + } + return ExprToAny(b.Balance(enable_variadic_logical_operators_)); +} + +std::any ParserVisitor::visitRelation(CelParser::RelationContext* ctx) { + if (ctx->calc()) { + return visit(ctx->calc()); + } + std::string op_text; + if (ctx->op) { + op_text = ctx->op->getText(); + } + auto op = ReverseLookupOperator(op_text); + if (op) { + auto lhs = ExprFromAny(visit(ctx->relation(0))); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + auto rhs = ExprFromAny(visit(ctx->relation(1))); + return ExprToAny( + GlobalCallOrMacro(op_id, *op, std::move(lhs), std::move(rhs))); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "operator not found")); +} + +std::any ParserVisitor::visitCalc(CelParser::CalcContext* ctx) { + if (ctx->unary()) { + return visit(ctx->unary()); + } + std::string op_text; + if (ctx->op) { + op_text = ctx->op->getText(); + } + auto op = ReverseLookupOperator(op_text); + if (op) { + auto lhs = ExprFromAny(visit(ctx->calc(0))); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + auto rhs = ExprFromAny(visit(ctx->calc(1))); + return ExprToAny( + GlobalCallOrMacro(op_id, *op, std::move(lhs), std::move(rhs))); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "operator not found")); +} + +std::any ParserVisitor::visitUnary(CelParser::UnaryContext* ctx) { + return ExprToAny(factory_.NewStringConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), "<>")); +} + +std::any ParserVisitor::visitLogicalNot(CelParser::LogicalNotContext* ctx) { + if (ctx->ops.size() % 2 == 0) { + return visit(ctx->member()); + } + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->ops[0])); + auto target = ExprFromAny(visit(ctx->member())); + return ExprToAny( + GlobalCallOrMacro(op_id, CelOperator::LOGICAL_NOT, std::move(target))); +} + +std::any ParserVisitor::visitNegate(CelParser::NegateContext* ctx) { + if (ctx->ops.size() % 2 == 0) { + return visit(ctx->member()); + } + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->ops[0])); + auto target = ExprFromAny(visit(ctx->member())); + return ExprToAny( + GlobalCallOrMacro(op_id, CelOperator::NEGATE, std::move(target))); +} + +std::string ParserVisitor::NormalizeIdentifier( + CelParser::EscapeIdentContext* ctx) { + if (auto* raw_id = tree_as(ctx); raw_id) { + return raw_id->id->getText(); + } + if (auto* escaped_id = tree_as(ctx); + escaped_id) { + if (!enable_quoted_identifiers_) { + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '`'"); + } + auto escaped_id_text = escaped_id->id->getText(); + return escaped_id_text.substr(1, escaped_id_text.size() - 2); + } + + // Fallthrough might occur if the parser is in an error state. + return ""; +} + +std::any ParserVisitor::visitSelect(CelParser::SelectContext* ctx) { + auto operand = ExprFromAny(visit(ctx->member())); + // Handle the error case where no valid identifier is specified. + if (!ctx->id || !ctx->op) { + return ExprToAny(factory_.NewUnspecified( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); + } + auto id = NormalizeIdentifier(ctx->id); + if (ctx->opt != nullptr) { + if (!enable_optional_syntax_) { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), "unsupported syntax '.?'")); + } + auto op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + std::vector arguments; + arguments.reserve(2); + arguments.push_back(std::move(operand)); + arguments.push_back(factory_.NewStringConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), std::move(id))); + return ExprToAny(factory_.NewCall(op_id, "_?._", std::move(arguments))); + } + return ExprToAny( + factory_.NewSelect(factory_.NextId(SourceRangeFromToken(ctx->op)), + std::move(operand), std::move(id))); +} + +std::any ParserVisitor::visitMemberCall(CelParser::MemberCallContext* ctx) { + auto operand = ExprFromAny(visit(ctx->member())); + // Handle the error case where no valid identifier is specified. + if (!ctx->id) { + return ExprToAny(factory_.NewUnspecified( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); + } + auto id = ctx->id->getText(); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->open)); + auto args = visitList(ctx->args); + return ExprToAny( + ReceiverCallOrMacroImpl(op_id, id, std::move(operand), std::move(args))); +} + +std::any ParserVisitor::visitIndex(CelParser::IndexContext* ctx) { + auto target = ExprFromAny(visit(ctx->member())); + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + auto index = ExprFromAny(visit(ctx->index)); + if (!enable_optional_syntax_ && ctx->opt != nullptr) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '.?'")); + } + return ExprToAny(GlobalCallOrMacro( + op_id, ctx->opt != nullptr ? "_[?_]" : CelOperator::INDEX, + std::move(target), std::move(index))); +} + +std::any ParserVisitor::visitCreateMessage( + CelParser::CreateMessageContext* ctx) { + std::vector parts; + parts.reserve(ctx->ids.size()); + for (const auto* id : ctx->ids) { + parts.push_back(id->getText()); + } + std::string name; + if (ctx->leadingDot) { + name.push_back('.'); + name.append(absl::StrJoin(parts, ".")); + } else { + name = absl::StrJoin(parts, "."); + } + int64_t obj_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + std::vector fields; + if (ctx->entries) { + fields = visitFields(ctx->entries); + } + return ExprToAny( + factory_.NewStruct(obj_id, std::move(name), std::move(fields))); +} + +std::any ParserVisitor::visitFieldInitializerList( + CelParser::FieldInitializerListContext* ctx) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "<>")); +} + +std::vector ParserVisitor::visitFields( + CelParser::FieldInitializerListContext* ctx) { + std::vector res; + if (!ctx || ctx->fields.empty()) { + return res; + } + + res.reserve(ctx->fields.size()); + for (size_t i = 0; i < ctx->fields.size(); ++i) { + if (i >= ctx->cols.size() || i >= ctx->values.size()) { + // This is the result of a syntax error detected elsewhere. + return res; + } + auto* f = ctx->fields[i]; + if (!f->escapeIdent()) { + ABSL_DCHECK(HasErrored()); + // This is the result of a syntax error detected elsewhere. + return res; + } + + std::string id = NormalizeIdentifier(f->escapeIdent()); + + int64_t init_id = factory_.NextId(SourceRangeFromToken(ctx->cols[i])); + if (!enable_optional_syntax_ && f->opt) { + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '?'"); + continue; + } + auto value = ExprFromAny(visit(ctx->values[i])); + res.push_back(factory_.NewStructField(init_id, std::move(id), + std::move(value), f->opt != nullptr)); + } + + return res; +} + +std::any ParserVisitor::visitIdent(CelParser::IdentContext* ctx) { + std::string ident_name; + if (ctx->leadingDot) { + ident_name = "."; + } + if (!ctx->id) { + return ExprToAny(factory_.NewUnspecified( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); + } + // check if ID is in reserved identifiers + if (cel::internal::LexisIsReserved(ctx->id->getText())) { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), + absl::StrFormat("reserved identifier: %s", ctx->id->getText()))); + } + + ident_name += ctx->id->getText(); + + return ExprToAny(factory_.NewIdent( + factory_.NextId(SourceRangeFromToken(ctx->id)), std::move(ident_name))); +} + +std::any ParserVisitor::visitGlobalCall(CelParser::GlobalCallContext* ctx) { + std::string ident_name; + if (ctx->leadingDot) { + ident_name = "."; + } + if (!ctx->id || !ctx->op) { + return ExprToAny(factory_.NewUnspecified( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); + } + // check if ID is in reserved identifiers + if (cel::internal::LexisIsReserved(ctx->id->getText())) { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), + absl::StrFormat("reserved identifier: %s", ctx->id->getText()))); + } + + ident_name += ctx->id->getText(); + + int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); + auto args = visitList(ctx->args); + return ExprToAny( + GlobalCallOrMacroImpl(op_id, std::move(ident_name), std::move(args))); +} + +std::any ParserVisitor::visitNested(CelParser::NestedContext* ctx) { + return visit(ctx->e); +} + +std::any ParserVisitor::visitCreateList(CelParser::CreateListContext* ctx) { + int64_t list_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + auto elems = visitList(ctx->elems); + return ExprToAny(factory_.NewList(list_id, std::move(elems))); +} + +std::vector ParserVisitor::visitList( + CelParser::ListInitContext* ctx) { + std::vector rv; + if (!ctx) return rv; + rv.reserve(ctx->elems.size()); + for (size_t i = 0; i < ctx->elems.size(); ++i) { + auto* expr_ctx = ctx->elems[i]; + if (expr_ctx == nullptr) { + return rv; + } + if (!enable_optional_syntax_ && expr_ctx->opt != nullptr) { + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '?'"); + // Still generate an ID to detect node limit exceeded. + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + rv.push_back(factory_.NewListElement(factory_.NewUnspecified(0), false)); + continue; + } + rv.push_back(factory_.NewListElement(ExprFromAny(visitExpr(expr_ctx->e)), + expr_ctx->opt != nullptr)); + } + return rv; +} + +std::vector ParserVisitor::visitList(CelParser::ExprListContext* ctx) { + std::vector rv; + if (!ctx) return rv; + std::transform(ctx->e.begin(), ctx->e.end(), std::back_inserter(rv), + [this](CelParser::ExprContext* expr_ctx) { + return ExprFromAny(visitExpr(expr_ctx)); + }); + return rv; +} + +std::any ParserVisitor::visitCreateMap(CelParser::CreateMapContext* ctx) { + int64_t struct_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + std::vector entries; + if (ctx->entries) { + entries = visitEntries(ctx->entries); + } + return ExprToAny(factory_.NewMap(struct_id, std::move(entries))); +} + +std::any ParserVisitor::visitConstantLiteral( + CelParser::ConstantLiteralContext* clctx) { + CelParser::LiteralContext* literal = clctx->literal(); + if (auto* ctx = tree_as(literal)) { + return visitInt(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitUint(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitDouble(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitString(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitBytes(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitBoolFalse(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitBoolTrue(ctx); + } else if (auto* ctx = tree_as(literal)) { + return visitNull(ctx); + } + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(clctx), + "invalid constant literal expression")); +} + +std::any ParserVisitor::visitMapInitializerList( + CelParser::MapInitializerListContext* ctx) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "<>")); +} + +std::vector ParserVisitor::visitEntries( + CelParser::MapInitializerListContext* ctx) { + std::vector res; + if (!ctx || ctx->keys.empty()) { + return res; + } + + res.reserve(ctx->cols.size()); + for (size_t i = 0; i < ctx->cols.size(); ++i) { + auto id = factory_.NextId(SourceRangeFromToken(ctx->cols[i])); + if (!enable_optional_syntax_ && ctx->keys[i]->opt) { + factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "unsupported syntax '?'"); + // Still generate an ID to detect node limit exceeded. + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + factory_.NextId(SourceRangeFromParserRuleContext(ctx)); + res.push_back(factory_.NewMapEntry(0, factory_.NewUnspecified(0), + factory_.NewUnspecified(0), false)); + continue; + } + auto key = ExprFromAny(visit(ctx->keys[i]->e)); + auto value = ExprFromAny(visit(ctx->values[i])); + res.push_back(factory_.NewMapEntry(id, std::move(key), std::move(value), + ctx->keys[i]->opt != nullptr)); + } + return res; +} + +std::any ParserVisitor::visitInt(CelParser::IntContext* ctx) { + std::string value; + if (ctx->sign) { + value = ctx->sign->getText(); + } + value += ctx->tok->getText(); + int64_t int_value; + if (absl::StartsWith(ctx->tok->getText(), "0x")) { + if (absl::SimpleHexAtoi(value, &int_value)) { + return ExprToAny(factory_.NewIntConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), int_value)); + } else { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), "invalid hex int literal")); + } + } + if (absl::SimpleAtoi(value, &int_value)) { + return ExprToAny(factory_.NewIntConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), int_value)); + } else { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "invalid int literal")); + } +} + +std::any ParserVisitor::visitUint(CelParser::UintContext* ctx) { + std::string value = ctx->tok->getText(); + // trim the 'u' designator included in the uint literal. + if (!value.empty()) { + value.resize(value.size() - 1); + } + uint64_t uint_value; + if (absl::StartsWith(ctx->tok->getText(), "0x")) { + if (absl::SimpleHexAtoi(value, &uint_value)) { + return ExprToAny(factory_.NewUintConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), uint_value)); + } else { + return ExprToAny(factory_.ReportError( + SourceRangeFromParserRuleContext(ctx), "invalid hex uint literal")); + } + } + if (absl::SimpleAtoi(value, &uint_value)) { + return ExprToAny(factory_.NewUintConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), uint_value)); + } else { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "invalid uint literal")); + } +} + +std::any ParserVisitor::visitDouble(CelParser::DoubleContext* ctx) { + std::string value; + if (ctx->sign) { + value = ctx->sign->getText(); + } + value += ctx->tok->getText(); + double double_value; + if (absl::SimpleAtod(value, &double_value)) { + return ExprToAny(factory_.NewDoubleConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), double_value)); + } else { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + "invalid double literal")); + } +} + +std::any ParserVisitor::visitString(CelParser::StringContext* ctx) { + auto status_or_value = cel::internal::ParseStringLiteral(ctx->tok->getText()); + if (!status_or_value.ok()) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + status_or_value.status().message())); + } + return ExprToAny(factory_.NewStringConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), + std::move(status_or_value).value())); +} + +std::any ParserVisitor::visitBytes(CelParser::BytesContext* ctx) { + auto status_or_value = cel::internal::ParseBytesLiteral(ctx->tok->getText()); + if (!status_or_value.ok()) { + return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), + status_or_value.status().message())); + } + return ExprToAny(factory_.NewBytesConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), + std::move(status_or_value).value())); +} + +std::any ParserVisitor::visitBoolTrue(CelParser::BoolTrueContext* ctx) { + return ExprToAny(factory_.NewBoolConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), true)); +} + +std::any ParserVisitor::visitBoolFalse(CelParser::BoolFalseContext* ctx) { + return ExprToAny(factory_.NewBoolConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)), false)); +} + +std::any ParserVisitor::visitNull(CelParser::NullContext* ctx) { + return ExprToAny(factory_.NewNullConst( + factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); +} + +cel::SourceInfo ParserVisitor::GetSourceInfo() { + cel::SourceInfo source_info; + source_info.set_location(std::string(source_.description())); + for (const auto& positions : factory_.positions()) { + source_info.mutable_positions().insert( + std::pair{positions.first, positions.second.begin}); + } + source_info.mutable_line_offsets().reserve(source_.line_offsets().size()); + for (const auto& line_offset : source_.line_offsets()) { + source_info.mutable_line_offsets().push_back(line_offset); + } + + source_info.mutable_macro_calls() = factory_.release_macro_calls(); + return source_info; +} + +EnrichedSourceInfo ParserVisitor::enriched_source_info() const { + absl::flat_hash_map> offsets; + offsets.reserve(factory_.positions().size()); + for (const auto& positions : factory_.positions()) { + offsets.insert( + std::pair{positions.first, + std::pair{positions.second.begin, positions.second.end - 1}}); + } + return EnrichedSourceInfo(std::move(offsets)); +} + +void ParserVisitor::syntaxError(antlr4::Recognizer* recognizer, + antlr4::Token* offending_symbol, size_t line, + size_t col, const std::string& msg, + std::exception_ptr e) { + cel::SourceRange range; + if (auto position = source_.GetPosition(cel::SourceLocation{ + static_cast(line), static_cast(col)}); + position) { + range.begin = *position; + } + factory_.ReportError(range, absl::StrCat("Syntax error: ", msg)); +} + +bool ParserVisitor::HasErrored() const { return factory_.HasErrors(); } + +std::vector ParserVisitor::CollectIssues() { + return factory_.CollectIssues(); +} + +Expr ParserVisitor::GlobalCallOrMacroImpl(int64_t expr_id, + absl::string_view function, + std::vector args) { + auto macro = macro_registry_.FindMacro(function, args.size(), false); + if (!macro) { + return factory_.NewCall(expr_id, function, std::move(args)); + } + if (factory_.is_node_limit_exceeded()) { + return factory_.ReportError( + factory_.GetSourceRange(expr_id), + "could not expand macro: expression node limit exceeded"); + } + std::vector macro_args; + if (add_macro_calls_) { + macro_args.reserve(args.size()); + for (const auto& arg : args) { + macro_args.push_back(factory_.BuildMacroCallArg(arg)); + } + } + factory_.BeginMacro(factory_.GetSourceRange(expr_id)); + auto expr = macro->Expand(factory_, std::nullopt, absl::MakeSpan(args)); + factory_.EndMacro(); + if (expr) { + if (add_macro_calls_) { + factory_.AddMacroCall(expr->id(), function, std::nullopt, + std::move(macro_args)); + } + // We did not end up using `expr_id`. Delete metadata. + factory_.EraseId(expr_id); + return std::move(*expr); + } + return factory_.NewCall(expr_id, function, std::move(args)); +} + +Expr ParserVisitor::ReceiverCallOrMacroImpl(int64_t expr_id, + absl::string_view function, + Expr target, + std::vector args) { + auto macro = macro_registry_.FindMacro(function, args.size(), true); + if (!macro) { + return factory_.NewMemberCall(expr_id, function, std::move(target), + std::move(args)); + } + if (factory_.is_node_limit_exceeded()) { + return factory_.ReportError( + factory_.GetSourceRange(expr_id), + "could not expand macro: expression node limit exceeded"); + } + + Expr macro_target; + std::vector macro_args; + if (add_macro_calls_) { + macro_args.reserve(args.size()); + macro_target = factory_.BuildMacroCallArg(target); + for (const auto& arg : args) { + macro_args.push_back(factory_.BuildMacroCallArg(arg)); + } + } + factory_.BeginMacro(factory_.GetSourceRange(expr_id)); + auto expr = macro->Expand(factory_, std::ref(target), absl::MakeSpan(args)); + factory_.EndMacro(); + if (expr) { + if (add_macro_calls_) { + factory_.AddMacroCall(expr->id(), function, std::move(macro_target), + std::move(macro_args)); + } + // We did not end up using `expr_id`. Delete metadata. + factory_.EraseId(expr_id); + return std::move(*expr); + } + + return factory_.NewMemberCall(expr_id, function, std::move(target), + std::move(args)); +} + +std::string ParserVisitor::ExtractQualifiedName(antlr4::ParserRuleContext* ctx, + const Expr& e) { + if (e == Expr{}) { + return ""; + } + + if (const auto* ident_expr = absl::get_if(&e.kind()); ident_expr) { + return ident_expr->name(); + } + if (const auto* select_expr = absl::get_if(&e.kind()); + select_expr) { + std::string prefix = ExtractQualifiedName(ctx, select_expr->operand()); + if (!prefix.empty()) { + return absl::StrCat(prefix, ".", select_expr->field()); + } + } + factory_.ReportError(factory_.GetSourceRange(e.id()), + "expected a qualified name"); + return ""; +} + +// Replacements for absl::StrReplaceAll for escaping standard whitespace +// characters. +static constexpr auto kStandardReplacements = + std::array, 3>{ + std::make_pair("\n", "\\n"), + std::make_pair("\r", "\\r"), + std::make_pair("\t", "\\t"), + }; + +static constexpr absl::string_view kSingleQuote = "'"; + +// ExprRecursionListener extends the standard ANTLR CelParser to ensure that +// recursive entries into the 'expr' rule are limited to a configurable depth so +// as to prevent stack overflows. +class ExprRecursionListener final : public ParseTreeListener { + public: + explicit ExprRecursionListener( + const int max_recursion_depth = kDefaultMaxRecursionDepth) + : max_recursion_depth_(max_recursion_depth), recursion_depth_(0) {} + ~ExprRecursionListener() override = default; + + void visitTerminal(TerminalNode* node) override {} + void visitErrorNode(ErrorNode* error) override {} + void enterEveryRule(ParserRuleContext* ctx) override; + void exitEveryRule(ParserRuleContext* ctx) override; + + private: + const int max_recursion_depth_; + int recursion_depth_; +}; + +void ExprRecursionListener::enterEveryRule(ParserRuleContext* ctx) { + // Throw a ParseCancellationException since the parsing would otherwise + // continue if this were treated as a syntax error and the problem would + // continue to manifest. + if (ctx->getRuleIndex() == CelParser::RuleExpr) { + if (recursion_depth_ > max_recursion_depth_) { + throw ParseCancellationException( + absl::StrFormat("Expression recursion limit exceeded. limit: %d", + max_recursion_depth_)); + } + recursion_depth_++; + } +} + +void ExprRecursionListener::exitEveryRule(ParserRuleContext* ctx) { + if (ctx->getRuleIndex() == CelParser::RuleExpr) { + recursion_depth_--; + } +} + +class RecoveryLimitErrorStrategy final : public DefaultErrorStrategy { + public: + explicit RecoveryLimitErrorStrategy( + int recovery_limit = kDefaultErrorRecoveryLimit, + int recovery_token_lookahead_limit = + kDefaultErrorRecoveryTokenLookaheadLimit) + : recovery_limit_(recovery_limit), + recovery_attempts_(0), + recovery_token_lookahead_limit_(recovery_token_lookahead_limit) {} + + void recover(Parser* recognizer, std::exception_ptr e) override { + checkRecoveryLimit(recognizer); + DefaultErrorStrategy::recover(recognizer, e); + } + + Token* recoverInline(Parser* recognizer) override { + checkRecoveryLimit(recognizer); + return DefaultErrorStrategy::recoverInline(recognizer); + } + + // Override the ANTLR implementation to introduce a token lookahead limit as + // this prevents pathologically constructed, yet small (< 16kb) inputs from + // consuming inordinate amounts of compute. + // + // This method is only called on error recovery paths. + void consumeUntil(Parser* recognizer, const IntervalSet& set) override { + size_t ttype = recognizer->getInputStream()->LA(1); + int recovery_search_depth = 0; + while (ttype != Token::EOF && !set.contains(ttype) && + recovery_search_depth++ < recovery_token_lookahead_limit_) { + recognizer->consume(); + ttype = recognizer->getInputStream()->LA(1); + } + // Halt all parsing if the lookahead limit is reached during error recovery. + if (recovery_search_depth == recovery_token_lookahead_limit_) { + throw ParseCancellationException("Unable to find a recovery token"); + } + } + + protected: + std::string escapeWSAndQuote(const std::string& s) const override { + std::string result; + result.reserve(s.size() + 2); + absl::StrAppend(&result, kSingleQuote, s, kSingleQuote); + absl::StrReplaceAll(kStandardReplacements, &result); + return result; + } + + private: + void checkRecoveryLimit(Parser* recognizer) { + if (recovery_attempts_++ >= recovery_limit_) { + std::string too_many_errors = + absl::StrFormat("More than %d parse errors.", recovery_limit_); + recognizer->notifyErrorListeners(too_many_errors); + throw ParseCancellationException(too_many_errors); + } + } + + int recovery_limit_; + int recovery_attempts_; + int recovery_token_lookahead_limit_; +}; + +} // namespace + +absl::StatusOr> AntlrParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, std::vector* parse_issues, + cel::EnrichedSourceInfo* enriched_source_info) { + ABSL_DCHECK(!options.enable_pratt_parser); + try { + CodePointStream input(source.content(), source.description()); + if (input.size() > options.expression_size_codepoint_limit) { + return absl::InvalidArgumentError(absl::StrCat( + "expression size exceeds codepoint limit.", " input size: ", + input.size(), ", limit: ", options.expression_size_codepoint_limit)); + } + CelLexer lexer(&input); + CommonTokenStream tokens(&lexer); + CelParser parser(&tokens); + ExprRecursionListener listener(options.max_recursion_depth); + ParserVisitor visitor( + source, options.max_recursion_depth, options.expression_node_limit, + registry, options.add_macro_calls, options.enable_optional_syntax, + options.enable_quoted_identifiers, + options.enable_variadic_logical_operators); + + lexer.removeErrorListeners(); + parser.removeErrorListeners(); + lexer.addErrorListener(&visitor); + parser.addErrorListener(&visitor); + parser.addParseListener(&listener); + + // Limit the number of error recovery attempts to prevent bad expressions + // from consuming lots of cpu / memory. + parser.setErrorHandler(std::make_shared( + options.error_recovery_limit, + options.error_recovery_token_lookahead_limit)); + + Expr expr; + try { + expr = ExprFromAny(visitor.visit(parser.start())); + } catch (const ParseCancellationException& e) { + if (visitor.HasErrored()) { + auto issues = visitor.CollectIssues(); + std::string error_message = FormatIssues(source, issues); + if (parse_issues != nullptr) { + *parse_issues = std::move(issues); + } + return absl::InvalidArgumentError(error_message); + } + return absl::CancelledError(e.what()); + } + + if (visitor.HasErrored()) { + auto issues = visitor.CollectIssues(); + std::string error_message = FormatIssues(source, issues); + if (parse_issues != nullptr) { + *parse_issues = std::move(issues); + } + return absl::InvalidArgumentError(error_message); + } + + if (enriched_source_info != nullptr) { + *enriched_source_info = visitor.enriched_source_info(); + } + + return std::make_unique(std::move(expr), visitor.GetSourceInfo()); + } catch (const std::exception& e) { + return absl::AbortedError(e.what()); + } catch (const char* what) { + // ANTLRv4 has historically thrown C string literals. + return absl::AbortedError(what); + } catch (...) { + // We guarantee to never throw and always return a status. + return absl::UnknownError("An unknown exception occurred"); + } +} + +absl::StatusOr> AntlrParserImpl::ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const { + return AntlrParseImpl(source, macro_registry_, options_, parse_issues); +} + +absl::StatusOr> AntlrParserImpl::PrepareSourceImpl( + absl::string_view input, absl::string_view description) const { + return cel::NewSource( + input, std::string(description), + cel::SourceOptions{.max_codepoint_size = + options_.expression_size_codepoint_limit}); +} + +absl::Status AntlrParserBuilderImpl::AddMacro(const cel::Macro& macro) { + for (const auto& existing_macro : macros_) { + if (existing_macro.key() == macro.key()) { + return absl::AlreadyExistsError( + absl::StrCat("macro already exists: ", macro.key())); + } + } + macros_.push_back(macro); + return absl::OkStatus(); +} + +absl::Status AntlrParserBuilderImpl::AddLibrary(cel::ParserLibrary library) { + if (!library.id.empty()) { + auto [it, inserted] = library_ids_.insert(library.id); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library already exists: ", library.id)); + } + } + libraries_.push_back(std::move(library)); + return absl::OkStatus(); +} + +absl::Status AntlrParserBuilderImpl::AddLibrarySubset( + cel::ParserLibrarySubset subset) { + if (subset.library_id.empty()) { + return absl::InvalidArgumentError("subset must have a library id"); + } + std::string library_id = subset.library_id; + auto [it, inserted] = + library_subsets_.insert({library_id, std::move(subset)}); + if (!inserted) { + return absl::AlreadyExistsError( + absl::StrCat("parser library subset already exists: ", library_id)); + } + return absl::OkStatus(); +} + +absl::StatusOr> AntlrParserBuilderImpl::Build() { + using std::swap; + // Save the old configured macros so they aren't affected by applying the + // libraries and can be restored if an error occurs. + std::vector individual_macros; + swap(individual_macros, macros_); + absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); + + cel::MacroRegistry macro_registry; + + for (const auto& library : libraries_) { + CEL_RETURN_IF_ERROR(library.configure(*this)); + if (!library.id.empty()) { + auto it = library_subsets_.find(library.id); + if (it != library_subsets_.end()) { + const cel::ParserLibrarySubset& subset = it->second; + for (const auto& macro : macros_) { + if (subset.should_include_macro(macro)) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); + } + } + macros_.clear(); + continue; + } + } + + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); + macros_.clear(); + } + + absl::flat_hash_set library_ids(library_ids_); + + // Hack to support adding the standard library macros either by option or + // with a library configurer. + if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); + library_ids.insert("stdlib"); + } + + if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); + library_ids.insert("optional"); + } + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); + return std::make_unique(options_, std::move(macro_registry), + std::move(library_ids)); +} + +std::unique_ptr AntlrParserImpl::ToBuilder() const { + auto ins = std::make_unique(options_); + ins->library_ids_ = library_ids_; + ins->macros_ = macro_registry_.ListMacros(); + return ins; +} + +} // namespace cel::parser_internal diff --git a/parser/internal/antlr_parser.h b/parser/internal/antlr_parser.h new file mode 100644 index 000000000..9e0153c6a --- /dev/null +++ b/parser/internal/antlr_parser.h @@ -0,0 +1,106 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_ANTLR_PARSER_H_ +#define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_ANTLR_PARSER_H_ + +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/source.h" +#include "parser/macro.h" +#include "parser/macro_registry.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel { +class EnrichedSourceInfo; +} // namespace cel + +namespace cel::parser_internal { + +class AntlrParserImpl final : public cel::Parser { + public: + explicit AntlrParserImpl(const cel::ParserOptions& options, + cel::MacroRegistry macro_registry, + absl::flat_hash_set library_ids) + : options_(options), + macro_registry_(std::move(macro_registry)), + library_ids_(std::move(library_ids)) {} + + ~AntlrParserImpl() override = default; + + absl::StatusOr> ParseImpl( + const cel::Source& source, + std::vector* absl_nullable parse_issues) const override; + + absl::StatusOr> PrepareSourceImpl( + absl::string_view input, absl::string_view description) const override; + + std::unique_ptr ToBuilder() const override; + + private: + cel::ParserOptions options_; + cel::MacroRegistry macro_registry_; + absl::flat_hash_set library_ids_; +}; + +absl::StatusOr> AntlrParseImpl( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options, + std::vector* parse_issues = nullptr, + cel::EnrichedSourceInfo* enriched_source_info = nullptr); + +class AntlrParserBuilderImpl final : public cel::ParserBuilder { + public: + explicit AntlrParserBuilderImpl(const cel::ParserOptions& options) + : options_(options) {} + + cel::ParserOptions& GetOptions() override { return options_; } + + absl::Status AddMacro(const cel::Macro& macro) override; + + absl::Status AddLibrary(cel::ParserLibrary library) override; + + absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override; + + absl::StatusOr> Build() override; + + private: + friend class AntlrParserImpl; + + cel::ParserOptions options_; + std::vector macros_; + absl::flat_hash_set library_ids_; + std::vector libraries_; + absl::flat_hash_map library_subsets_; +}; + +inline std::unique_ptr NewAntlrParserBuilder( + const cel::ParserOptions& options = cel::ParserOptions()) { + return std::make_unique(options); +} + +} // namespace cel::parser_internal + +#endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_ANTLR_PARSER_H_ diff --git a/parser/internal/antlr_parser_test.cc b/parser/internal/antlr_parser_test.cc new file mode 100644 index 000000000..d1a99227a --- /dev/null +++ b/parser/internal/antlr_parser_test.cc @@ -0,0 +1,76 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "parser/internal/antlr_parser.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/ast.h" +#include "common/source.h" +#include "internal/status_macros.h" +#include "internal/testing.h" +#include "parser/options.h" +#include "parser/parser_interface.h" + +namespace cel::parser_internal { +namespace { + +using ::absl_testing::IsOk; +using ::testing::HasSubstr; +using ::testing::Not; + +absl::StatusOr> Parse( + absl::string_view expression, absl::string_view description = "", + const cel::ParserOptions& options = cel::ParserOptions(), + std::vector* issues = nullptr) { + std::unique_ptr builder = NewAntlrParserBuilder(options); + CEL_ASSIGN_OR_RETURN(std::unique_ptr parser, builder->Build()); + CEL_ASSIGN_OR_RETURN(std::unique_ptr source, + cel::NewSource(expression, std::string(description))); + return parser->Parse(*source, issues); +} + +TEST(AntlrParserTest, ErrorRecoveryLimits) { + ParserOptions options; + options.error_recovery_limit = 1; + auto result = Parse("......", "", options); + EXPECT_THAT(result, Not(IsOk())); + EXPECT_EQ(result.status().message(), + "ERROR: :1:1: Syntax error: More than 1 parse errors.\n | ......\n " + "| ^\nERROR: :1:2: Syntax error: no viable alternative at input " + "'..'\n | ......\n | .^"); +} + +TEST(AntlrParserTest, RecursionDepthExceeded) { + ParserOptions options; + // AST visitor will recurse a variable amount depending on the terms used in + // the expression. This check occurs in the business logic converting the raw + // Antlr parse tree into an Expr. There is a separate check (via a custom + // listener) for AST depth while running the antlr generated parser. + options.max_recursion_depth = 6; + auto result = Parse("1 + 2 + 3 + 4 + 5 + 6 + 7", "", options); + + EXPECT_THAT(result, Not(IsOk())); + EXPECT_THAT(result.status().message(), + HasSubstr("Exceeded max recursion depth of 6 when parsing.")); +} + +} // namespace +} // namespace cel::parser_internal diff --git a/parser/parser.cc b/parser/parser.cc index 9f94c3a91..37c74365d 100644 --- a/parser/parser.cc +++ b/parser/parser.cc @@ -14,1905 +14,89 @@ #include "parser/parser.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include -#include #include #include #include "cel/expr/syntax.pb.h" -#include "absl/base/macros.h" -#include "absl/base/optimization.h" -#include "absl/cleanup/cleanup.h" -#include "absl/container/btree_map.h" -#include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" -#include "absl/functional/overload.h" -#include "absl/log/absl_check.h" -#include "absl/log/check.h" -#include "absl/memory/memory.h" -#include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/match.h" -#include "absl/strings/numbers.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "absl/strings/str_join.h" -#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "absl/types/span.h" -#include "absl/types/variant.h" -#include "antlr4-runtime.h" #include "common/ast.h" #include "common/ast/expr_proto.h" #include "common/ast/source_info_proto.h" -#include "common/constant.h" -#include "common/expr.h" -#include "common/expr_factory.h" -#include "common/operators.h" #include "common/source.h" -#include "internal/lexis.h" #include "internal/status_macros.h" -#include "internal/strings.h" +#ifndef EXCLUDE_CEL_ANTLR_PARSER +#include "parser/internal/antlr_parser.h" +#endif #include "parser/internal/pratt_parser.h" -#pragma push_macro("IN") -#undef IN -#include "parser/internal/CelBaseVisitor.h" -#include "parser/internal/CelLexer.h" -#include "parser/internal/CelParser.h" -#pragma pop_macro("IN") #include "parser/macro.h" -#include "parser/macro_expr_factory.h" #include "parser/macro_registry.h" #include "parser/options.h" #include "parser/parser_interface.h" #include "parser/source_factory.h" namespace google::api::expr::parser { -namespace { -class ParserVisitor; -} -} // namespace google::api::expr::parser - -namespace cel { - -namespace { - -constexpr const char kHiddenAccumulatorVariableName[] = "@result"; - -std::any ExprPtrToAny(std::unique_ptr&& expr) { - return std::make_any(expr.release()); -} - -std::any ExprToAny(Expr&& expr) { - return ExprPtrToAny(std::make_unique(std::move(expr))); -} - -std::unique_ptr ExprPtrFromAny(std::any&& any) { - return absl::WrapUnique(std::any_cast(std::move(any))); -} - -Expr ExprFromAny(std::any&& any) { - auto expr = ExprPtrFromAny(std::move(any)); - return std::move(*expr); -} - -struct ParserError { - std::string message; - SourceRange range; -}; - -std::string DisplayParserError(const cel::Source& source, - SourceLocation location, - absl::string_view message) { - return absl::StrCat(absl::StrFormat("ERROR: %s:%zu:%zu: %s", - source.description(), location.line, - // add one to the 0-based column - location.column + 1, message), - source.DisplayErrorLocation(location)); -} - -int32_t PositiveOrMax(int32_t value) { - return value >= 0 ? value : std::numeric_limits::max(); -} - -SourceRange SourceRangeFromToken(const antlr4::Token* token) { - SourceRange range; - if (token != nullptr) { - if (auto start = token->getStartIndex(); start != INVALID_INDEX) { - range.begin = static_cast(start); - } - if (auto end = token->getStopIndex(); end != INVALID_INDEX) { - range.end = static_cast(end + 1); - } - } - return range; -} - -SourceRange SourceRangeFromParserRuleContext( - const antlr4::ParserRuleContext* context) { - SourceRange range; - if (context != nullptr) { - if (auto start = context->getStart() != nullptr - ? context->getStart()->getStartIndex() - : INVALID_INDEX; - start != INVALID_INDEX) { - range.begin = static_cast(start); - } - if (auto end = context->getStop() != nullptr - ? context->getStop()->getStopIndex() - : INVALID_INDEX; - end != INVALID_INDEX) { - range.end = static_cast(end + 1); - } - } - return range; -} - -} // namespace - -class ParserMacroExprFactory final : public MacroExprFactory { - public: - explicit ParserMacroExprFactory(const cel::Source& source, - int expression_node_limit) - : source_(source), expression_node_limit_(expression_node_limit) {} - void BeginMacro(SourceRange macro_position) { - macro_position_ = macro_position; - } - - void EndMacro() { macro_position_ = SourceRange{}; } - - Expr ReportError(absl::string_view message) override { - return ReportError(macro_position_, message); - } - - Expr ReportError(int64_t expr_id, absl::string_view message) { - return ReportError(GetSourceRange(expr_id), message); - } - - Expr ReportError(SourceRange range, absl::string_view message) { - ++error_count_; - if (errors_.size() <= 100) { - errors_.push_back(ParserError{std::string(message), range}); - } - return NewUnspecified(NextId(range)); - } - - Expr ReportErrorAt(const Expr& expr, absl::string_view message) override { - return ReportError(GetSourceRange(expr.id()), message); - } - - SourceRange GetSourceRange(int64_t id) const { - if (auto it = positions_.find(id); it != positions_.end()) { - return it->second; - } - return SourceRange{}; - } - - int64_t NextId(const SourceRange& range) { - auto id = expr_id_++; - if (id > expression_node_limit_ && !node_limit_exceeded_) { - node_limit_exceeded_ = true; - ReportError(range, "expression node limit exceeded"); - } - if (range.begin != -1 || range.end != -1) { - positions_.insert(std::pair{id, range}); - } - return id; - } - - bool is_node_limit_exceeded() const { return node_limit_exceeded_; } - - bool HasErrors() const { return error_count_ != 0; } - - std::vector CollectIssues() { - // Errors are collected as they are encountered, not by their location - // within the source. To have a more stable error message as implementation - // details change, we sort the collected errors by their source location - // first. - std::stable_sort( - errors_.begin(), errors_.end(), - [](const ParserError& lhs, const ParserError& rhs) -> bool { - auto lhs_begin = PositiveOrMax(lhs.range.begin); - auto lhs_end = PositiveOrMax(lhs.range.end); - auto rhs_begin = PositiveOrMax(rhs.range.begin); - auto rhs_end = PositiveOrMax(rhs.range.end); - return lhs_begin < rhs_begin || - (lhs_begin == rhs_begin && lhs_end < rhs_end); - }); - // Build the summary error message using the sorted errors. - bool errors_truncated = error_count_ > 100; - std::vector issues; - issues.reserve( - errors_.size() + - errors_truncated); // Reserve space for the transform and an - // additional element when truncation occurs. - std::transform( - errors_.begin(), errors_.end(), std::back_inserter(issues), - [this](const ParserError& error) { - auto location = - source_.GetLocation(error.range.begin).value_or(SourceLocation{}); - return cel::ParseIssue(location, error.message); - }); - if (errors_truncated) { - issues.push_back(cel::ParseIssue( - absl::StrCat(error_count_ - 100, " more errors were truncated."))); - } - return issues; - } - - void AddMacroCall(int64_t macro_id, absl::string_view function, - absl::optional target, std::vector arguments) { - macro_calls_.insert( - {macro_id, target.has_value() - ? NewMemberCall(0, function, std::move(*target), - std::move(arguments)) - : NewCall(0, function, std::move(arguments))}); - } - - Expr BuildMacroCallArg(const Expr& expr) { - if (auto it = macro_calls_.find(expr.id()); it != macro_calls_.end()) { - return NewUnspecified(expr.id()); - } - return absl::visit( - absl::Overload( - [this, &expr](const UnspecifiedExpr&) -> Expr { - return NewUnspecified(expr.id()); - }, - [this, &expr](const Constant& const_expr) -> Expr { - return NewConst(expr.id(), const_expr); - }, - [this, &expr](const IdentExpr& ident_expr) -> Expr { - return NewIdent(expr.id(), ident_expr.name()); - }, - [this, &expr](const SelectExpr& select_expr) -> Expr { - return select_expr.test_only() - ? NewPresenceTest( - expr.id(), - BuildMacroCallArg(select_expr.operand()), - select_expr.field()) - : NewSelect(expr.id(), - BuildMacroCallArg(select_expr.operand()), - select_expr.field()); - }, - [this, &expr](const CallExpr& call_expr) -> Expr { - std::vector macro_arguments; - macro_arguments.reserve(call_expr.args().size()); - for (const auto& argument : call_expr.args()) { - macro_arguments.push_back(BuildMacroCallArg(argument)); - } - absl::optional macro_target; - if (call_expr.has_target()) { - macro_target = BuildMacroCallArg(call_expr.target()); - } - return macro_target.has_value() - ? NewMemberCall(expr.id(), call_expr.function(), - std::move(*macro_target), - std::move(macro_arguments)) - : NewCall(expr.id(), call_expr.function(), - std::move(macro_arguments)); - }, - [this, &expr](const ListExpr& list_expr) -> Expr { - std::vector macro_elements; - macro_elements.reserve(list_expr.elements().size()); - for (const auto& element : list_expr.elements()) { - auto& cloned_element = macro_elements.emplace_back(); - if (element.has_expr()) { - cloned_element.set_expr(BuildMacroCallArg(element.expr())); - } - cloned_element.set_optional(element.optional()); - } - return NewList(expr.id(), std::move(macro_elements)); - }, - [this, &expr](const StructExpr& struct_expr) -> Expr { - std::vector macro_fields; - macro_fields.reserve(struct_expr.fields().size()); - for (const auto& field : struct_expr.fields()) { - auto& macro_field = macro_fields.emplace_back(); - macro_field.set_id(field.id()); - macro_field.set_name(field.name()); - macro_field.set_value(BuildMacroCallArg(field.value())); - macro_field.set_optional(field.optional()); - } - return NewStruct(expr.id(), struct_expr.name(), - std::move(macro_fields)); - }, - [this, &expr](const MapExpr& map_expr) -> Expr { - std::vector macro_entries; - macro_entries.reserve(map_expr.entries().size()); - for (const auto& entry : map_expr.entries()) { - auto& macro_entry = macro_entries.emplace_back(); - macro_entry.set_id(entry.id()); - macro_entry.set_key(BuildMacroCallArg(entry.key())); - macro_entry.set_value(BuildMacroCallArg(entry.value())); - macro_entry.set_optional(entry.optional()); - } - return NewMap(expr.id(), std::move(macro_entries)); - }, - [this, &expr](const ComprehensionExpr& comprehension_expr) -> Expr { - return NewComprehension( - expr.id(), comprehension_expr.iter_var(), - BuildMacroCallArg(comprehension_expr.iter_range()), - comprehension_expr.accu_var(), - BuildMacroCallArg(comprehension_expr.accu_init()), - BuildMacroCallArg(comprehension_expr.loop_condition()), - BuildMacroCallArg(comprehension_expr.loop_step()), - BuildMacroCallArg(comprehension_expr.result())); - }), - expr.kind()); - } - - using ExprFactory::NewBoolConst; - using ExprFactory::NewBytesConst; - using ExprFactory::NewCall; - using ExprFactory::NewComprehension; - using ExprFactory::NewConst; - using ExprFactory::NewDoubleConst; - using ExprFactory::NewIdent; - using ExprFactory::NewIntConst; - using ExprFactory::NewList; - using ExprFactory::NewListElement; - using ExprFactory::NewMap; - using ExprFactory::NewMapEntry; - using ExprFactory::NewMemberCall; - using ExprFactory::NewNullConst; - using ExprFactory::NewPresenceTest; - using ExprFactory::NewSelect; - using ExprFactory::NewStringConst; - using ExprFactory::NewStruct; - using ExprFactory::NewStructField; - using ExprFactory::NewUintConst; - using ExprFactory::NewUnspecified; - - const absl::btree_map& positions() const { - return positions_; - } - - const absl::flat_hash_map& macro_calls() const { - return macro_calls_; - } - - absl::flat_hash_map release_macro_calls() { - using std::swap; - absl::flat_hash_map result; - swap(result, macro_calls_); - return result; - } - - void EraseId(ExprId id) { - positions_.erase(id); - if (expr_id_ == id + 1) { - --expr_id_; - } - } - - protected: - int64_t NextId() override { return NextId(macro_position_); } - - int64_t CopyId(int64_t id) override { - if (id == 0) { - return 0; - } - return NextId(GetSourceRange(id)); - } - - private: - int64_t expr_id_ = 1; - absl::btree_map positions_; - absl::flat_hash_map macro_calls_; - std::vector errors_; - size_t error_count_ = 0; - const Source& source_; - int expression_node_limit_; - bool node_limit_exceeded_ = false; - SourceRange macro_position_; -}; - -} // namespace cel - -namespace google::api::expr::parser { - -namespace { - -using ::antlr4::CharStream; -using ::antlr4::CommonTokenStream; -using ::antlr4::DefaultErrorStrategy; -using ::antlr4::ParseCancellationException; -using ::antlr4::Parser; -using ::antlr4::ParserRuleContext; -using ::antlr4::Token; -using ::antlr4::misc::IntervalSet; -using ::antlr4::tree::ErrorNode; -using ::antlr4::tree::ParseTreeListener; -using ::antlr4::tree::TerminalNode; -using ::cel::Expr; -using ::cel::ExprFromAny; -using ::cel::ExprKind; -using ::cel::ExprToAny; -using ::cel::IdentExpr; -using ::cel::ListExprElement; -using ::cel::MapExprEntry; -using ::cel::SelectExpr; -using ::cel::SourceRangeFromParserRuleContext; -using ::cel::SourceRangeFromToken; -using ::cel::StructExprField; -using ::cel_parser_internal::CelBaseVisitor; -using ::cel_parser_internal::CelLexer; -using ::cel_parser_internal::CelParser; -using common::CelOperator; -using common::ReverseLookupOperator; +using ::cel::Macro; +using ::cel::ParserOptions; using ::cel::expr::ParsedExpr; +using ::google::api::expr::parser::EnrichedSourceInfo; +using ::google::api::expr::parser::VerboseParsedExpr; -class CodePointStream final : public CharStream { - public: - CodePointStream(cel::SourceContentView buffer, absl::string_view source_name) - : buffer_(buffer), - source_name_(source_name), - size_(buffer_.size()), - index_(0) {} - - void consume() override { - if (ABSL_PREDICT_FALSE(index_ >= size_)) { - ABSL_ASSERT(LA(1) == IntStream::EOF); - throw antlr4::IllegalStateException("cannot consume EOF"); - } - index_++; - } - - size_t LA(ptrdiff_t i) override { - if (ABSL_PREDICT_FALSE(i == 0)) { - return 0; - } - auto p = static_cast(index_); - if (i < 0) { - i++; - if (p + i - 1 < 0) { - return IntStream::EOF; - } - } - if (p + i - 1 >= static_cast(size_)) { - return IntStream::EOF; - } - return buffer_.at(static_cast(p + i - 1)); - } - - ptrdiff_t mark() override { return -1; } - - void release(ptrdiff_t marker) override {} - - size_t index() override { return index_; } - - void seek(size_t index) override { index_ = std::min(index, size_); } - - size_t size() override { return size_; } - - std::string getSourceName() const override { - return source_name_.empty() ? IntStream::UNKNOWN_SOURCE_NAME - : std::string(source_name_); - } - - std::string getText(const antlr4::misc::Interval& interval) override { - if (ABSL_PREDICT_FALSE(interval.a < 0 || interval.b < 0)) { - return std::string(); - } - size_t start = static_cast(interval.a); - if (ABSL_PREDICT_FALSE(start >= size_)) { - return std::string(); - } - size_t stop = static_cast(interval.b); - if (ABSL_PREDICT_FALSE(stop >= size_)) { - stop = size_ - 1; - } - return buffer_.ToString(static_cast(start), - static_cast(stop) + 1); - } - - std::string toString() const override { return buffer_.ToString(); } - - private: - cel::SourceContentView const buffer_; - const absl::string_view source_name_; - const size_t size_; - size_t index_; -}; - -// Scoped helper for incrementing the parse recursion count. -// Increments on creation, decrements on destruction (stack unwind). -class ScopedIncrement final { - public: - explicit ScopedIncrement(int& recursion_depth) - : recursion_depth_(recursion_depth) { - ++recursion_depth_; - } - - ~ScopedIncrement() { --recursion_depth_; } - - private: - int& recursion_depth_; -}; - -// balancer performs tree balancing on operators whose arguments are of equal -// precedence. -// -// The purpose of the balancer is to ensure a compact serialization format for -// the logical &&, || operators which have a tendency to create long DAGs which -// are skewed in one direction. Since the operators are commutative re-ordering -// the terms *must not* affect the evaluation result. -// -// Based on code from //third_party/cel/go/parser/helper.go -class ExpressionBalancer final { - public: - ExpressionBalancer(cel::ParserMacroExprFactory& factory, std::string function, - Expr expr); - - // addTerm adds an operation identifier and term to the set of terms to be - // balanced. - void AddTerm(int64_t op, Expr term); - - // balance creates a balanced tree from the sub-terms and returns the final - // Expr value. - Expr Balance(bool enable_variadic = false); - - private: - // balancedTree recursively balances the terms provided to a commutative - // operator. - Expr BalancedTree(int lo, int hi); - - private: - cel::ParserMacroExprFactory& factory_; - std::string function_; - std::vector terms_; - std::vector ops_; -}; - -ExpressionBalancer::ExpressionBalancer(cel::ParserMacroExprFactory& factory, - std::string function, Expr expr) - : factory_(factory), function_(std::move(function)) { - terms_.push_back(std::move(expr)); -} - -void ExpressionBalancer::AddTerm(int64_t op, Expr term) { - terms_.push_back(std::move(term)); - ops_.push_back(op); -} - -Expr ExpressionBalancer::Balance(bool enable_variadic) { - if (terms_.size() == 1) { - return std::move(terms_[0]); - } - if (enable_variadic) { - return factory_.NewCall(ops_[0], function_, std::move(terms_)); - } - return BalancedTree(0, ops_.size() - 1); -} - -Expr ExpressionBalancer::BalancedTree(int lo, int hi) { - int mid = (lo + hi + 1) / 2; - - std::vector arguments; - arguments.reserve(2); - - if (mid == lo) { - arguments.push_back(std::move(terms_[mid])); - } else { - arguments.push_back(BalancedTree(lo, mid - 1)); - } - - if (mid == hi) { - arguments.push_back(std::move(terms_[mid + 1])); - } else { - arguments.push_back(BalancedTree(mid + 1, hi)); - } - return factory_.NewCall(ops_[mid], function_, std::move(arguments)); -} - -std::string FormatIssues(const cel::Source& source, - absl::Span issues) { - return absl::StrJoin( - issues, "\n", [&source](std::string* out, const cel::ParseIssue& issue) { - absl::StrAppend(out, cel::DisplayParserError(source, issue.location(), - issue.message())); - }); -} - -class ParserVisitor final : public CelBaseVisitor, - public antlr4::BaseErrorListener { - public: - ParserVisitor(const cel::Source& source, int max_recursion_depth, - int max_expression_node_count, - const cel::MacroRegistry& macro_registry, - bool add_macro_calls = false, - bool enable_optional_syntax = false, - bool enable_quoted_identifiers = false, - bool enable_variadic_logical_operators = false) - : source_(source), - factory_(source_, max_expression_node_count), - macro_registry_(macro_registry), - recursion_depth_(0), - max_recursion_depth_(max_recursion_depth), - add_macro_calls_(add_macro_calls), - enable_optional_syntax_(enable_optional_syntax), - enable_quoted_identifiers_(enable_quoted_identifiers), - enable_variadic_logical_operators_(enable_variadic_logical_operators) {} - - ~ParserVisitor() override = default; - - std::any visit(antlr4::tree::ParseTree* tree) override; - - std::any visitStart(CelParser::StartContext* ctx) override; - std::any visitExpr(CelParser::ExprContext* ctx) override; - std::any visitConditionalOr(CelParser::ConditionalOrContext* ctx) override; - std::any visitConditionalAnd(CelParser::ConditionalAndContext* ctx) override; - std::any visitRelation(CelParser::RelationContext* ctx) override; - std::any visitCalc(CelParser::CalcContext* ctx) override; - std::any visitUnary(CelParser::UnaryContext* ctx); - std::any visitLogicalNot(CelParser::LogicalNotContext* ctx) override; - std::any visitNegate(CelParser::NegateContext* ctx) override; - std::any visitSelect(CelParser::SelectContext* ctx) override; - std::any visitMemberCall(CelParser::MemberCallContext* ctx) override; - std::any visitIndex(CelParser::IndexContext* ctx) override; - std::any visitCreateMessage(CelParser::CreateMessageContext* ctx) override; - std::any visitFieldInitializerList( - CelParser::FieldInitializerListContext* ctx) override; - std::vector visitFields( - CelParser::FieldInitializerListContext* ctx); - std::any visitGlobalCall(CelParser::GlobalCallContext* ctx) override; - std::any visitIdent(CelParser::IdentContext* ctx) override; - std::any visitNested(CelParser::NestedContext* ctx) override; - std::any visitCreateList(CelParser::CreateListContext* ctx) override; - std::vector visitList(CelParser::ListInitContext* ctx); - std::vector visitList(CelParser::ExprListContext* ctx); - std::any visitCreateMap(CelParser::CreateMapContext* ctx) override; - std::any visitConstantLiteral( - CelParser::ConstantLiteralContext* ctx) override; - std::any visitPrimaryExpr(CelParser::PrimaryExprContext* ctx) override; - std::any visitMemberExpr(CelParser::MemberExprContext* ctx) override; - - std::any visitMapInitializerList( - CelParser::MapInitializerListContext* ctx) override; - std::vector visitEntries( - CelParser::MapInitializerListContext* ctx); - std::any visitInt(CelParser::IntContext* ctx) override; - std::any visitUint(CelParser::UintContext* ctx) override; - std::any visitDouble(CelParser::DoubleContext* ctx) override; - std::any visitString(CelParser::StringContext* ctx) override; - std::any visitBytes(CelParser::BytesContext* ctx) override; - std::any visitBoolTrue(CelParser::BoolTrueContext* ctx) override; - std::any visitBoolFalse(CelParser::BoolFalseContext* ctx) override; - std::any visitNull(CelParser::NullContext* ctx) override; - // Note: this is destructive and intended to be called after the parse is - // finished. - cel::SourceInfo GetSourceInfo(); - EnrichedSourceInfo enriched_source_info() const; - void syntaxError(antlr4::Recognizer* recognizer, - antlr4::Token* offending_symbol, size_t line, size_t col, - const std::string& msg, std::exception_ptr e) override; - bool HasErrored() const; - - std::vector CollectIssues(); - - private: - template - Expr GlobalCallOrMacro(int64_t expr_id, absl::string_view function, - Args&&... args) { - std::vector arguments; - arguments.reserve(sizeof...(Args)); - (arguments.push_back(std::forward(args)), ...); - return GlobalCallOrMacroImpl(expr_id, function, std::move(arguments)); - } - - Expr GlobalCallOrMacroImpl(int64_t expr_id, absl::string_view function, - std::vector args); - Expr ReceiverCallOrMacroImpl(int64_t expr_id, absl::string_view function, - Expr target, std::vector args); - std::string ExtractQualifiedName(antlr4::ParserRuleContext* ctx, - const Expr& e); - - std::string NormalizeIdentifier(CelParser::EscapeIdentContext* ctx); - // Attempt to unnest parse context. - // - // Walk the parse tree to the first complex term to reduce recursive depth in - // the visit* calls. - antlr4::tree::ParseTree* UnnestContext(antlr4::tree::ParseTree* tree); - - private: - const cel::Source& source_; - cel::ParserMacroExprFactory factory_; - const cel::MacroRegistry& macro_registry_; - int recursion_depth_; - const int max_recursion_depth_; - const bool add_macro_calls_; - const bool enable_optional_syntax_; - const bool enable_quoted_identifiers_; - const bool enable_variadic_logical_operators_; -}; - -template ::value>> -T* tree_as(antlr4::tree::ParseTree* tree) { - return dynamic_cast(tree); -} - -std::any ParserVisitor::visit(antlr4::tree::ParseTree* tree) { - ScopedIncrement inc(recursion_depth_); - if (recursion_depth_ > max_recursion_depth_) { - return ExprToAny(factory_.ReportError( - absl::StrFormat("Exceeded max recursion depth of %d when parsing.", - max_recursion_depth_))); - } - tree = UnnestContext(tree); - if (auto* ctx = tree_as(tree)) { - return visitStart(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitExpr(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitConditionalAnd(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitConditionalOr(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitRelation(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitCalc(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitLogicalNot(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitPrimaryExpr(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitMemberExpr(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitSelect(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitMemberCall(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitMapInitializerList(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitNegate(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitIndex(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitUnary(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitCreateList(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitCreateMessage(ctx); - } else if (auto* ctx = tree_as(tree)) { - return visitCreateMap(ctx); - } - - if (tree) { - return ExprToAny( - factory_.ReportError(SourceRangeFromParserRuleContext( - tree_as(tree)), - "unknown parsetree type")); - } - return ExprToAny(factory_.ReportError("<> parsetree")); -} - -std::any ParserVisitor::visitPrimaryExpr(CelParser::PrimaryExprContext* pctx) { - CelParser::PrimaryContext* primary = pctx->primary(); - if (auto* ctx = tree_as(primary)) { - return visitNested(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitIdent(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitGlobalCall(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitCreateList(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitCreateMap(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitCreateMessage(ctx); - } else if (auto* ctx = tree_as(primary)) { - return visitConstantLiteral(ctx); - } - if (factory_.HasErrors()) { - // ANTLR creates PrimaryContext rather than a derived class during certain - // error conditions. This is odd, but we ignore it as we already have errors - // that occurred. - return ExprToAny(factory_.NewUnspecified(factory_.NextId({}))); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(pctx), - "invalid primary expression")); -} - -std::any ParserVisitor::visitMemberExpr(CelParser::MemberExprContext* mctx) { - CelParser::MemberContext* member = mctx->member(); - if (auto* ctx = tree_as(member)) { - return visitPrimaryExpr(ctx); - } else if (auto* ctx = tree_as(member)) { - return visitSelect(ctx); - } else if (auto* ctx = tree_as(member)) { - return visitMemberCall(ctx); - } else if (auto* ctx = tree_as(member)) { - return visitIndex(ctx); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(mctx), - "unsupported simple expression")); -} - -std::any ParserVisitor::visitStart(CelParser::StartContext* ctx) { - return visit(ctx->expr()); -} - -antlr4::tree::ParseTree* ParserVisitor::UnnestContext( - antlr4::tree::ParseTree* tree) { - antlr4::tree::ParseTree* last = nullptr; - while (tree != last) { - last = tree; - - if (auto* ctx = tree_as(tree)) { - tree = ctx->expr(); - } - - if (auto* ctx = tree_as(tree)) { - if (ctx->op != nullptr) { - return ctx; - } - tree = ctx->e; - } - - if (auto* ctx = tree_as(tree)) { - if (!ctx->ops.empty()) { - return ctx; - } - tree = ctx->e; - } - - if (auto* ctx = tree_as(tree)) { - if (!ctx->ops.empty()) { - return ctx; - } - tree = ctx->e; - } - - if (auto* ctx = tree_as(tree)) { - if (ctx->calc() == nullptr) { - return ctx; - } - tree = ctx->calc(); - } - - if (auto* ctx = tree_as(tree)) { - if (ctx->unary() == nullptr) { - return ctx; - } - tree = ctx->unary(); - } - - if (auto* ctx = tree_as(tree)) { - tree = ctx->member(); - } - - if (auto* ctx = tree_as(tree)) { - if (auto* nested = tree_as(ctx->primary())) { - tree = nested->e; - } else { - return ctx; - } - } - } - - return tree; -} - -std::any ParserVisitor::visitExpr(CelParser::ExprContext* ctx) { - auto result = ExprFromAny(visit(ctx->e)); - if (!ctx->op) { - return ExprToAny(std::move(result)); - } - std::vector arguments; - arguments.reserve(3); - arguments.push_back(std::move(result)); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - arguments.push_back(ExprFromAny(visit(ctx->e1))); - arguments.push_back(ExprFromAny(visit(ctx->e2))); - - return ExprToAny( - factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(arguments))); -} - -std::any ParserVisitor::visitConditionalOr( - CelParser::ConditionalOrContext* ctx) { - auto result = ExprFromAny(visit(ctx->e)); - if (ctx->ops.empty()) { - return ExprToAny(std::move(result)); - } - ExpressionBalancer b(factory_, CelOperator::LOGICAL_OR, std::move(result)); - for (size_t i = 0; i < ctx->ops.size(); ++i) { - auto op = ctx->ops[i]; - if (i >= ctx->e1.size()) { - return ExprToAny( - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unexpected character, wanted '||'")); - } - auto next = ExprFromAny(visit(ctx->e1[i])); - int64_t op_id = factory_.NextId(SourceRangeFromToken(op)); - b.AddTerm(op_id, std::move(next)); - } - return ExprToAny(b.Balance(enable_variadic_logical_operators_)); -} - -std::any ParserVisitor::visitConditionalAnd( - CelParser::ConditionalAndContext* ctx) { - auto result = ExprFromAny(visit(ctx->e)); - if (ctx->ops.empty()) { - return ExprToAny(std::move(result)); - } - ExpressionBalancer b(factory_, CelOperator::LOGICAL_AND, std::move(result)); - for (size_t i = 0; i < ctx->ops.size(); ++i) { - auto op = ctx->ops[i]; - if (i >= ctx->e1.size()) { - return ExprToAny( - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unexpected character, wanted '&&'")); - } - auto next = ExprFromAny(visit(ctx->e1[i])); - int64_t op_id = factory_.NextId(SourceRangeFromToken(op)); - b.AddTerm(op_id, std::move(next)); - } - return ExprToAny(b.Balance(enable_variadic_logical_operators_)); -} - -std::any ParserVisitor::visitRelation(CelParser::RelationContext* ctx) { - if (ctx->calc()) { - return visit(ctx->calc()); - } - std::string op_text; - if (ctx->op) { - op_text = ctx->op->getText(); - } - auto op = ReverseLookupOperator(op_text); - if (op) { - auto lhs = ExprFromAny(visit(ctx->relation(0))); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto rhs = ExprFromAny(visit(ctx->relation(1))); - return ExprToAny( - GlobalCallOrMacro(op_id, *op, std::move(lhs), std::move(rhs))); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "operator not found")); -} - -std::any ParserVisitor::visitCalc(CelParser::CalcContext* ctx) { - if (ctx->unary()) { - return visit(ctx->unary()); - } - std::string op_text; - if (ctx->op) { - op_text = ctx->op->getText(); - } - auto op = ReverseLookupOperator(op_text); - if (op) { - auto lhs = ExprFromAny(visit(ctx->calc(0))); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto rhs = ExprFromAny(visit(ctx->calc(1))); - return ExprToAny( - GlobalCallOrMacro(op_id, *op, std::move(lhs), std::move(rhs))); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "operator not found")); -} - -std::any ParserVisitor::visitUnary(CelParser::UnaryContext* ctx) { - return ExprToAny(factory_.NewStringConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), "<>")); -} - -std::any ParserVisitor::visitLogicalNot(CelParser::LogicalNotContext* ctx) { - if (ctx->ops.size() % 2 == 0) { - return visit(ctx->member()); - } - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->ops[0])); - auto target = ExprFromAny(visit(ctx->member())); - return ExprToAny( - GlobalCallOrMacro(op_id, CelOperator::LOGICAL_NOT, std::move(target))); -} - -std::any ParserVisitor::visitNegate(CelParser::NegateContext* ctx) { - if (ctx->ops.size() % 2 == 0) { - return visit(ctx->member()); - } - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->ops[0])); - auto target = ExprFromAny(visit(ctx->member())); - return ExprToAny( - GlobalCallOrMacro(op_id, CelOperator::NEGATE, std::move(target))); -} - -std::string ParserVisitor::NormalizeIdentifier( - CelParser::EscapeIdentContext* ctx) { - if (auto* raw_id = tree_as(ctx); raw_id) { - return raw_id->id->getText(); - } - if (auto* escaped_id = tree_as(ctx); - escaped_id) { - if (!enable_quoted_identifiers_) { - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '`'"); - } - auto escaped_id_text = escaped_id->id->getText(); - return escaped_id_text.substr(1, escaped_id_text.size() - 2); - } - - // Fallthrough might occur if the parser is in an error state. - return ""; -} - -std::any ParserVisitor::visitSelect(CelParser::SelectContext* ctx) { - auto operand = ExprFromAny(visit(ctx->member())); - // Handle the error case where no valid identifier is specified. - if (!ctx->id || !ctx->op) { - return ExprToAny(factory_.NewUnspecified( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); - } - auto id = NormalizeIdentifier(ctx->id); - if (ctx->opt != nullptr) { - if (!enable_optional_syntax_) { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), "unsupported syntax '.?'")); - } - auto op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - std::vector arguments; - arguments.reserve(2); - arguments.push_back(std::move(operand)); - arguments.push_back(factory_.NewStringConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), std::move(id))); - return ExprToAny(factory_.NewCall(op_id, "_?._", std::move(arguments))); - } - return ExprToAny( - factory_.NewSelect(factory_.NextId(SourceRangeFromToken(ctx->op)), - std::move(operand), std::move(id))); -} - -std::any ParserVisitor::visitMemberCall(CelParser::MemberCallContext* ctx) { - auto operand = ExprFromAny(visit(ctx->member())); - // Handle the error case where no valid identifier is specified. - if (!ctx->id) { - return ExprToAny(factory_.NewUnspecified( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); - } - auto id = ctx->id->getText(); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->open)); - auto args = visitList(ctx->args); - return ExprToAny( - ReceiverCallOrMacroImpl(op_id, id, std::move(operand), std::move(args))); -} - -std::any ParserVisitor::visitIndex(CelParser::IndexContext* ctx) { - auto target = ExprFromAny(visit(ctx->member())); - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto index = ExprFromAny(visit(ctx->index)); - if (!enable_optional_syntax_ && ctx->opt != nullptr) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '.?'")); - } - return ExprToAny(GlobalCallOrMacro( - op_id, ctx->opt != nullptr ? "_[?_]" : CelOperator::INDEX, - std::move(target), std::move(index))); -} - -std::any ParserVisitor::visitCreateMessage( - CelParser::CreateMessageContext* ctx) { - std::vector parts; - parts.reserve(ctx->ids.size()); - for (const auto* id : ctx->ids) { - parts.push_back(id->getText()); - } - std::string name; - if (ctx->leadingDot) { - name.push_back('.'); - name.append(absl::StrJoin(parts, ".")); - } else { - name = absl::StrJoin(parts, "."); - } - int64_t obj_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); - std::vector fields; - if (ctx->entries) { - fields = visitFields(ctx->entries); - } - return ExprToAny( - factory_.NewStruct(obj_id, std::move(name), std::move(fields))); -} - -std::any ParserVisitor::visitFieldInitializerList( - CelParser::FieldInitializerListContext* ctx) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "<>")); -} - -std::vector ParserVisitor::visitFields( - CelParser::FieldInitializerListContext* ctx) { - std::vector res; - if (!ctx || ctx->fields.empty()) { - return res; - } - - res.reserve(ctx->fields.size()); - for (size_t i = 0; i < ctx->fields.size(); ++i) { - if (i >= ctx->cols.size() || i >= ctx->values.size()) { - // This is the result of a syntax error detected elsewhere. - return res; - } - auto* f = ctx->fields[i]; - if (!f->escapeIdent()) { - ABSL_DCHECK(HasErrored()); - // This is the result of a syntax error detected elsewhere. - return res; - } - - std::string id = NormalizeIdentifier(f->escapeIdent()); - - int64_t init_id = factory_.NextId(SourceRangeFromToken(ctx->cols[i])); - if (!enable_optional_syntax_ && f->opt) { - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '?'"); - continue; - } - auto value = ExprFromAny(visit(ctx->values[i])); - res.push_back(factory_.NewStructField(init_id, std::move(id), - std::move(value), f->opt != nullptr)); - } - - return res; -} - -std::any ParserVisitor::visitIdent(CelParser::IdentContext* ctx) { - std::string ident_name; - if (ctx->leadingDot) { - ident_name = "."; - } - if (!ctx->id) { - return ExprToAny(factory_.NewUnspecified( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); - } - // check if ID is in reserved identifiers - if (cel::internal::LexisIsReserved(ctx->id->getText())) { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), - absl::StrFormat("reserved identifier: %s", ctx->id->getText()))); - } - - ident_name += ctx->id->getText(); - - return ExprToAny(factory_.NewIdent( - factory_.NextId(SourceRangeFromToken(ctx->id)), std::move(ident_name))); -} - -std::any ParserVisitor::visitGlobalCall(CelParser::GlobalCallContext* ctx) { - std::string ident_name; - if (ctx->leadingDot) { - ident_name = "."; - } - if (!ctx->id || !ctx->op) { - return ExprToAny(factory_.NewUnspecified( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); - } - // check if ID is in reserved identifiers - if (cel::internal::LexisIsReserved(ctx->id->getText())) { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), - absl::StrFormat("reserved identifier: %s", ctx->id->getText()))); - } - - ident_name += ctx->id->getText(); - - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->op)); - auto args = visitList(ctx->args); - return ExprToAny( - GlobalCallOrMacroImpl(op_id, std::move(ident_name), std::move(args))); -} - -std::any ParserVisitor::visitNested(CelParser::NestedContext* ctx) { - return visit(ctx->e); -} - -std::any ParserVisitor::visitCreateList(CelParser::CreateListContext* ctx) { - int64_t list_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); - auto elems = visitList(ctx->elems); - return ExprToAny(factory_.NewList(list_id, std::move(elems))); -} - -std::vector ParserVisitor::visitList( - CelParser::ListInitContext* ctx) { - std::vector rv; - if (!ctx) return rv; - rv.reserve(ctx->elems.size()); - for (size_t i = 0; i < ctx->elems.size(); ++i) { - auto* expr_ctx = ctx->elems[i]; - if (expr_ctx == nullptr) { - return rv; - } - if (!enable_optional_syntax_ && expr_ctx->opt != nullptr) { - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '?'"); - // Still generate an ID to detect node limit exceeded. - factory_.NextId(SourceRangeFromParserRuleContext(ctx)); - rv.push_back(factory_.NewListElement(factory_.NewUnspecified(0), false)); - continue; - } - rv.push_back(factory_.NewListElement(ExprFromAny(visitExpr(expr_ctx->e)), - expr_ctx->opt != nullptr)); - } - return rv; -} - -std::vector ParserVisitor::visitList(CelParser::ExprListContext* ctx) { - std::vector rv; - if (!ctx) return rv; - std::transform(ctx->e.begin(), ctx->e.end(), std::back_inserter(rv), - [this](CelParser::ExprContext* expr_ctx) { - return ExprFromAny(visitExpr(expr_ctx)); - }); - return rv; -} - -std::any ParserVisitor::visitCreateMap(CelParser::CreateMapContext* ctx) { - int64_t struct_id = factory_.NextId(SourceRangeFromParserRuleContext(ctx)); - std::vector entries; - if (ctx->entries) { - entries = visitEntries(ctx->entries); - } - return ExprToAny(factory_.NewMap(struct_id, std::move(entries))); -} - -std::any ParserVisitor::visitConstantLiteral( - CelParser::ConstantLiteralContext* clctx) { - CelParser::LiteralContext* literal = clctx->literal(); - if (auto* ctx = tree_as(literal)) { - return visitInt(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitUint(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitDouble(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitString(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitBytes(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitBoolFalse(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitBoolTrue(ctx); - } else if (auto* ctx = tree_as(literal)) { - return visitNull(ctx); - } - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(clctx), - "invalid constant literal expression")); -} - -std::any ParserVisitor::visitMapInitializerList( - CelParser::MapInitializerListContext* ctx) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "<>")); -} - -std::vector ParserVisitor::visitEntries( - CelParser::MapInitializerListContext* ctx) { - std::vector res; - if (!ctx || ctx->keys.empty()) { - return res; - } - - res.reserve(ctx->cols.size()); - for (size_t i = 0; i < ctx->cols.size(); ++i) { - auto id = factory_.NextId(SourceRangeFromToken(ctx->cols[i])); - if (!enable_optional_syntax_ && ctx->keys[i]->opt) { - factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "unsupported syntax '?'"); - // Still generate an ID to detect node limit exceeded. - factory_.NextId(SourceRangeFromParserRuleContext(ctx)); - factory_.NextId(SourceRangeFromParserRuleContext(ctx)); - res.push_back(factory_.NewMapEntry(0, factory_.NewUnspecified(0), - factory_.NewUnspecified(0), false)); - continue; - } - auto key = ExprFromAny(visit(ctx->keys[i]->e)); - auto value = ExprFromAny(visit(ctx->values[i])); - res.push_back(factory_.NewMapEntry(id, std::move(key), std::move(value), - ctx->keys[i]->opt != nullptr)); - } - return res; -} - -std::any ParserVisitor::visitInt(CelParser::IntContext* ctx) { - std::string value; - if (ctx->sign) { - value = ctx->sign->getText(); - } - value += ctx->tok->getText(); - int64_t int_value; - if (absl::StartsWith(ctx->tok->getText(), "0x")) { - if (absl::SimpleHexAtoi(value, &int_value)) { - return ExprToAny(factory_.NewIntConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), int_value)); - } else { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), "invalid hex int literal")); - } - } - if (absl::SimpleAtoi(value, &int_value)) { - return ExprToAny(factory_.NewIntConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), int_value)); - } else { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "invalid int literal")); - } -} - -std::any ParserVisitor::visitUint(CelParser::UintContext* ctx) { - std::string value = ctx->tok->getText(); - // trim the 'u' designator included in the uint literal. - if (!value.empty()) { - value.resize(value.size() - 1); - } - uint64_t uint_value; - if (absl::StartsWith(ctx->tok->getText(), "0x")) { - if (absl::SimpleHexAtoi(value, &uint_value)) { - return ExprToAny(factory_.NewUintConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), uint_value)); - } else { - return ExprToAny(factory_.ReportError( - SourceRangeFromParserRuleContext(ctx), "invalid hex uint literal")); - } - } - if (absl::SimpleAtoi(value, &uint_value)) { - return ExprToAny(factory_.NewUintConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), uint_value)); - } else { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "invalid uint literal")); - } -} - -std::any ParserVisitor::visitDouble(CelParser::DoubleContext* ctx) { - std::string value; - if (ctx->sign) { - value = ctx->sign->getText(); - } - value += ctx->tok->getText(); - double double_value; - if (absl::SimpleAtod(value, &double_value)) { - return ExprToAny(factory_.NewDoubleConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), double_value)); +absl::StatusOr EnrichedParse( + const cel::Source& source, const cel::MacroRegistry& registry, + const ParserOptions& options) { + ParsedExpr parsed_expr; + EnrichedSourceInfo enriched_source_info; + std::unique_ptr ast; +#ifdef EXCLUDE_CEL_ANTLR_PARSER + CEL_ASSIGN_OR_RETURN( + ast, cel::parser_internal::PrattParseImpl(source, registry, options, + /*parse_issues=*/nullptr, + &enriched_source_info)); +#else + if (options.enable_pratt_parser) { + CEL_ASSIGN_OR_RETURN( + ast, cel::parser_internal::PrattParseImpl(source, registry, options, + /*parse_issues=*/nullptr, + &enriched_source_info)); } else { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - "invalid double literal")); - } -} - -std::any ParserVisitor::visitString(CelParser::StringContext* ctx) { - auto status_or_value = cel::internal::ParseStringLiteral(ctx->tok->getText()); - if (!status_or_value.ok()) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - status_or_value.status().message())); - } - return ExprToAny(factory_.NewStringConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), - std::move(status_or_value).value())); -} - -std::any ParserVisitor::visitBytes(CelParser::BytesContext* ctx) { - auto status_or_value = cel::internal::ParseBytesLiteral(ctx->tok->getText()); - if (!status_or_value.ok()) { - return ExprToAny(factory_.ReportError(SourceRangeFromParserRuleContext(ctx), - status_or_value.status().message())); - } - return ExprToAny(factory_.NewBytesConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), - std::move(status_or_value).value())); -} - -std::any ParserVisitor::visitBoolTrue(CelParser::BoolTrueContext* ctx) { - return ExprToAny(factory_.NewBoolConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), true)); -} - -std::any ParserVisitor::visitBoolFalse(CelParser::BoolFalseContext* ctx) { - return ExprToAny(factory_.NewBoolConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)), false)); -} - -std::any ParserVisitor::visitNull(CelParser::NullContext* ctx) { - return ExprToAny(factory_.NewNullConst( - factory_.NextId(SourceRangeFromParserRuleContext(ctx)))); -} - -cel::SourceInfo ParserVisitor::GetSourceInfo() { - cel::SourceInfo source_info; - source_info.set_location(std::string(source_.description())); - for (const auto& positions : factory_.positions()) { - source_info.mutable_positions().insert( - std::pair{positions.first, positions.second.begin}); - } - source_info.mutable_line_offsets().reserve(source_.line_offsets().size()); - for (const auto& line_offset : source_.line_offsets()) { - source_info.mutable_line_offsets().push_back(line_offset); - } - - source_info.mutable_macro_calls() = factory_.release_macro_calls(); - return source_info; -} - -EnrichedSourceInfo ParserVisitor::enriched_source_info() const { - absl::flat_hash_map> offsets; - offsets.reserve(factory_.positions().size()); - for (const auto& positions : factory_.positions()) { - offsets.insert( - std::pair{positions.first, - std::pair{positions.second.begin, positions.second.end - 1}}); - } - return EnrichedSourceInfo(std::move(offsets)); -} - -void ParserVisitor::syntaxError(antlr4::Recognizer* recognizer, - antlr4::Token* offending_symbol, size_t line, - size_t col, const std::string& msg, - std::exception_ptr e) { - cel::SourceRange range; - if (auto position = source_.GetPosition(cel::SourceLocation{ - static_cast(line), static_cast(col)}); - position) { - range.begin = *position; - } - factory_.ReportError(range, absl::StrCat("Syntax error: ", msg)); -} - -bool ParserVisitor::HasErrored() const { return factory_.HasErrors(); } - -std::vector ParserVisitor::CollectIssues() { - return factory_.CollectIssues(); -} - -Expr ParserVisitor::GlobalCallOrMacroImpl(int64_t expr_id, - absl::string_view function, - std::vector args) { - auto macro = macro_registry_.FindMacro(function, args.size(), false); - if (!macro) { - return factory_.NewCall(expr_id, function, std::move(args)); - } - if (factory_.is_node_limit_exceeded()) { - return factory_.ReportError( - factory_.GetSourceRange(expr_id), - "could not expand macro: expression node limit exceeded"); - } - std::vector macro_args; - if (add_macro_calls_) { - macro_args.reserve(args.size()); - for (const auto& arg : args) { - macro_args.push_back(factory_.BuildMacroCallArg(arg)); - } - } - factory_.BeginMacro(factory_.GetSourceRange(expr_id)); - auto expr = macro->Expand(factory_, std::nullopt, absl::MakeSpan(args)); - factory_.EndMacro(); - if (expr) { - if (add_macro_calls_) { - factory_.AddMacroCall(expr->id(), function, std::nullopt, - std::move(macro_args)); - } - // We did not end up using `expr_id`. Delete metadata. - factory_.EraseId(expr_id); - return std::move(*expr); - } - return factory_.NewCall(expr_id, function, std::move(args)); -} - -Expr ParserVisitor::ReceiverCallOrMacroImpl(int64_t expr_id, - absl::string_view function, - Expr target, - std::vector args) { - auto macro = macro_registry_.FindMacro(function, args.size(), true); - if (!macro) { - return factory_.NewMemberCall(expr_id, function, std::move(target), - std::move(args)); - } - if (factory_.is_node_limit_exceeded()) { - return factory_.ReportError( - factory_.GetSourceRange(expr_id), - "could not expand macro: expression node limit exceeded"); - } - - Expr macro_target; - std::vector macro_args; - if (add_macro_calls_) { - macro_args.reserve(args.size()); - macro_target = factory_.BuildMacroCallArg(target); - for (const auto& arg : args) { - macro_args.push_back(factory_.BuildMacroCallArg(arg)); - } - } - factory_.BeginMacro(factory_.GetSourceRange(expr_id)); - auto expr = macro->Expand(factory_, std::ref(target), absl::MakeSpan(args)); - factory_.EndMacro(); - if (expr) { - if (add_macro_calls_) { - factory_.AddMacroCall(expr->id(), function, std::move(macro_target), - std::move(macro_args)); - } - // We did not end up using `expr_id`. Delete metadata. - factory_.EraseId(expr_id); - return std::move(*expr); - } - - return factory_.NewMemberCall(expr_id, function, std::move(target), - std::move(args)); -} - -std::string ParserVisitor::ExtractQualifiedName(antlr4::ParserRuleContext* ctx, - const Expr& e) { - if (e == Expr{}) { - return ""; - } - - if (const auto* ident_expr = absl::get_if(&e.kind()); ident_expr) { - return ident_expr->name(); - } - if (const auto* select_expr = absl::get_if(&e.kind()); - select_expr) { - std::string prefix = ExtractQualifiedName(ctx, select_expr->operand()); - if (!prefix.empty()) { - return absl::StrCat(prefix, ".", select_expr->field()); - } - } - factory_.ReportError(factory_.GetSourceRange(e.id()), - "expected a qualified name"); - return ""; -} - -// Replacements for absl::StrReplaceAll for escaping standard whitespace -// characters. -static constexpr auto kStandardReplacements = - std::array, 3>{ - std::make_pair("\n", "\\n"), - std::make_pair("\r", "\\r"), - std::make_pair("\t", "\\t"), - }; - -static constexpr absl::string_view kSingleQuote = "'"; - -// ExprRecursionListener extends the standard ANTLR CelParser to ensure that -// recursive entries into the 'expr' rule are limited to a configurable depth so -// as to prevent stack overflows. -class ExprRecursionListener final : public ParseTreeListener { - public: - explicit ExprRecursionListener( - const int max_recursion_depth = kDefaultMaxRecursionDepth) - : max_recursion_depth_(max_recursion_depth), recursion_depth_(0) {} - ~ExprRecursionListener() override {} - - void visitTerminal(TerminalNode* node) override {}; - void visitErrorNode(ErrorNode* error) override {}; - void enterEveryRule(ParserRuleContext* ctx) override; - void exitEveryRule(ParserRuleContext* ctx) override; - - private: - const int max_recursion_depth_; - int recursion_depth_; -}; - -void ExprRecursionListener::enterEveryRule(ParserRuleContext* ctx) { - // Throw a ParseCancellationException since the parsing would otherwise - // continue if this were treated as a syntax error and the problem would - // continue to manifest. - if (ctx->getRuleIndex() == CelParser::RuleExpr) { - if (recursion_depth_ > max_recursion_depth_) { - throw ParseCancellationException( - absl::StrFormat("Expression recursion limit exceeded. limit: %d", - max_recursion_depth_)); - } - recursion_depth_++; + CEL_ASSIGN_OR_RETURN( + ast, cel::parser_internal::AntlrParseImpl(source, registry, options, + /*parse_issues=*/nullptr, + &enriched_source_info)); } -} +#endif -void ExprRecursionListener::exitEveryRule(ParserRuleContext* ctx) { - if (ctx->getRuleIndex() == CelParser::RuleExpr) { - recursion_depth_--; - } + CEL_RETURN_IF_ERROR(cel::ast_internal::ExprToProto( + ast->root_expr(), parsed_expr.mutable_expr())); + CEL_RETURN_IF_ERROR(cel::ast_internal::SourceInfoToProto( + ast->source_info(), parsed_expr.mutable_source_info())); + return VerboseParsedExpr(std::move(parsed_expr), + std::move(enriched_source_info)); } -class RecoveryLimitErrorStrategy final : public DefaultErrorStrategy { - public: - explicit RecoveryLimitErrorStrategy( - int recovery_limit = kDefaultErrorRecoveryLimit, - int recovery_token_lookahead_limit = - kDefaultErrorRecoveryTokenLookaheadLimit) - : recovery_limit_(recovery_limit), - recovery_attempts_(0), - recovery_token_lookahead_limit_(recovery_token_lookahead_limit) {} - - void recover(Parser* recognizer, std::exception_ptr e) override { - checkRecoveryLimit(recognizer); - DefaultErrorStrategy::recover(recognizer, e); - } - - Token* recoverInline(Parser* recognizer) override { - checkRecoveryLimit(recognizer); - return DefaultErrorStrategy::recoverInline(recognizer); - } - - // Override the ANTLR implementation to introduce a token lookahead limit as - // this prevents pathologically constructed, yet small (< 16kb) inputs from - // consuming inordinate amounts of compute. - // - // This method is only called on error recovery paths. - void consumeUntil(Parser* recognizer, const IntervalSet& set) override { - size_t ttype = recognizer->getInputStream()->LA(1); - int recovery_search_depth = 0; - while (ttype != Token::EOF && !set.contains(ttype) && - recovery_search_depth++ < recovery_token_lookahead_limit_) { - recognizer->consume(); - ttype = recognizer->getInputStream()->LA(1); - } - // Halt all parsing if the lookahead limit is reached during error recovery. - if (recovery_search_depth == recovery_token_lookahead_limit_) { - throw ParseCancellationException("Unable to find a recovery token"); - } - } - - protected: - std::string escapeWSAndQuote(const std::string& s) const override { - std::string result; - result.reserve(s.size() + 2); - absl::StrAppend(&result, kSingleQuote, s, kSingleQuote); - absl::StrReplaceAll(kStandardReplacements, &result); - return result; - } - - private: - void checkRecoveryLimit(Parser* recognizer) { - if (recovery_attempts_++ >= recovery_limit_) { - std::string too_many_errors = - absl::StrFormat("More than %d parse errors.", recovery_limit_); - recognizer->notifyErrorListeners(too_many_errors); - throw ParseCancellationException(too_many_errors); - } - } - - int recovery_limit_; - int recovery_attempts_; - int recovery_token_lookahead_limit_; -}; - -struct ParseResult { - cel::Expr expr; - cel::SourceInfo source_info; - EnrichedSourceInfo enriched_source_info; -}; - -absl::StatusOr ParseImpl( - const cel::Source& source, const cel::MacroRegistry& registry, - const ParserOptions& options, - std::vector* parse_issues = nullptr) { - ABSL_DCHECK(!options.enable_pratt_parser); - try { - CodePointStream input(source.content(), source.description()); - if (input.size() > options.expression_size_codepoint_limit) { - return absl::InvalidArgumentError(absl::StrCat( - "expression size exceeds codepoint limit.", " input size: ", - input.size(), ", limit: ", options.expression_size_codepoint_limit)); - } - CelLexer lexer(&input); - CommonTokenStream tokens(&lexer); - CelParser parser(&tokens); - ExprRecursionListener listener(options.max_recursion_depth); - ParserVisitor visitor( - source, options.max_recursion_depth, options.expression_node_limit, - registry, options.add_macro_calls, options.enable_optional_syntax, - options.enable_quoted_identifiers, - options.enable_variadic_logical_operators); - - lexer.removeErrorListeners(); - parser.removeErrorListeners(); - lexer.addErrorListener(&visitor); - parser.addErrorListener(&visitor); - parser.addParseListener(&listener); - - // Limit the number of error recovery attempts to prevent bad expressions - // from consuming lots of cpu / memory. - parser.setErrorHandler(std::make_shared( - options.error_recovery_limit, - options.error_recovery_token_lookahead_limit)); - - Expr expr; - try { - expr = ExprFromAny(visitor.visit(parser.start())); - } catch (const ParseCancellationException& e) { - if (visitor.HasErrored()) { - auto issues = visitor.CollectIssues(); - std::string error_message = FormatIssues(source, issues); - if (parse_issues != nullptr) { - *parse_issues = std::move(issues); - } - return absl::InvalidArgumentError(error_message); - } - return absl::CancelledError(e.what()); - } - - if (visitor.HasErrored()) { - auto issues = visitor.CollectIssues(); - std::string error_message = FormatIssues(source, issues); - if (parse_issues != nullptr) { - *parse_issues = std::move(issues); - } - return absl::InvalidArgumentError(error_message); - } - - return { - ParseResult{.expr = std::move(expr), - .source_info = visitor.GetSourceInfo(), - .enriched_source_info = visitor.enriched_source_info()}}; - } catch (const std::exception& e) { - return absl::AbortedError(e.what()); - } catch (const char* what) { - // ANTLRv4 has historically thrown C string literals. - return absl::AbortedError(what); - } catch (...) { - // We guarantee to never throw and always return a status. - return absl::UnknownError("An unknown exception occurred"); - } +absl::StatusOr EnrichedParse( + absl::string_view expression, const std::vector& macros, + absl::string_view description, const ParserOptions& options) { + CEL_ASSIGN_OR_RETURN(auto source, + cel::NewSource(expression, std::string(description))); + cel::MacroRegistry macro_registry; + CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros)); + return EnrichedParse(*source, macro_registry, options); } -class ParserImpl : public cel::Parser { - public: - explicit ParserImpl(const ParserOptions& options, - cel::MacroRegistry macro_registry, - absl::flat_hash_set library_ids) - : options_(options), - macro_registry_(std::move(macro_registry)), - library_ids_(std::move(library_ids)) {} - - absl::StatusOr> ParseImpl( - const cel::Source& source, - std::vector* parse_issues) const override { - CEL_ASSIGN_OR_RETURN(auto parse_result, - ::google::api::expr::parser::ParseImpl( - source, macro_registry_, options_, parse_issues)); - return std::make_unique(std::move(parse_result.expr), - std::move(parse_result.source_info)); - } - - absl::StatusOr> PrepareSourceImpl( - absl::string_view input, absl::string_view description) const override { - return cel::NewSource( - input, std::string(description), - cel::SourceOptions{.max_codepoint_size = - options_.expression_size_codepoint_limit}); - } - - std::unique_ptr ToBuilder() const override; - - private: - const ParserOptions options_; - const cel::MacroRegistry macro_registry_; - absl::flat_hash_set library_ids_; -}; - -class ParserBuilderImpl : public cel::ParserBuilder { - public: - explicit ParserBuilderImpl(const ParserOptions& options) - : options_(options) {} - - ParserOptions& GetOptions() override { return options_; } - - absl::Status AddMacro(const cel::Macro& macro) override { - for (const auto& existing_macro : macros_) { - if (existing_macro.key() == macro.key()) { - return absl::AlreadyExistsError( - absl::StrCat("macro already exists: ", macro.key())); - } - } - macros_.push_back(macro); - return absl::OkStatus(); - } - - absl::Status AddLibrary(cel::ParserLibrary library) override { - if (!library.id.empty()) { - auto [it, inserted] = library_ids_.insert(library.id); - if (!inserted) { - return absl::AlreadyExistsError( - absl::StrCat("parser library already exists: ", library.id)); - } - } - libraries_.push_back(std::move(library)); - return absl::OkStatus(); - } - - absl::Status AddLibrarySubset(cel::ParserLibrarySubset subset) override { - if (subset.library_id.empty()) { - return absl::InvalidArgumentError("subset must have a library id"); - } - std::string library_id = subset.library_id; - auto [it, inserted] = - library_subsets_.insert({library_id, std::move(subset)}); - if (!inserted) { - return absl::AlreadyExistsError( - absl::StrCat("parser library subset already exists: ", library_id)); - } - return absl::OkStatus(); - } - - absl::StatusOr> Build() override { - using std::swap; - // Save the old configured macros so they aren't affected by applying the - // libraries and can be restored if an error occurs. - std::vector individual_macros; - swap(individual_macros, macros_); - absl::Cleanup cleanup([&] { swap(macros_, individual_macros); }); - - cel::MacroRegistry macro_registry; - - for (const auto& library : libraries_) { - CEL_RETURN_IF_ERROR(library.configure(*this)); - if (!library.id.empty()) { - auto it = library_subsets_.find(library.id); - if (it != library_subsets_.end()) { - const cel::ParserLibrarySubset& subset = it->second; - for (const auto& macro : macros_) { - if (subset.should_include_macro(macro)) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(macro)); - } - } - macros_.clear(); - continue; - } - } - - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros_)); - macros_.clear(); - } - - absl::flat_hash_set library_ids(library_ids_); - - // Hack to support adding the standard library macros either by option or - // with a library configurer. - if (!options_.disable_standard_macros && !library_ids_.contains("stdlib")) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(Macro::AllMacros())); - library_ids.insert("stdlib"); - } - - if (options_.enable_optional_syntax && !library_ids_.contains("optional")) { - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptMapMacro())); - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacro(cel::OptFlatMapMacro())); - library_ids.insert("optional"); - } - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(individual_macros)); - if (options_.enable_pratt_parser) { - return std::make_unique( - options_, std::move(macro_registry), std::move(library_ids)); - } - return std::make_unique(options_, std::move(macro_registry), - std::move(library_ids)); - } - - private: - friend class ParserImpl; - - ParserOptions options_; - std::vector macros_; - absl::flat_hash_set library_ids_; - std::vector libraries_; - absl::flat_hash_map library_subsets_; -}; - -std::unique_ptr ParserImpl::ToBuilder() const { - auto ins = std::make_unique(options_); - ins->library_ids_ = library_ids_; - ins->macros_ = macro_registry_.ListMacros(); - return ins; +absl::StatusOr ParseWithMacros(absl::string_view expression, + const std::vector& macros, + absl::string_view description, + const ParserOptions& options) { + CEL_ASSIGN_OR_RETURN(auto verbose_parsed_expr, + EnrichedParse(expression, macros, description, options)); + return verbose_parsed_expr.parsed_expr(); } -} // namespace - absl::StatusOr Parse(absl::string_view expression, absl::string_view description, const ParserOptions& options) { @@ -1927,54 +111,7 @@ absl::StatusOr Parse(absl::string_view expression, return ParseWithMacros(expression, macros, description, options); } -absl::StatusOr ParseWithMacros(absl::string_view expression, - const std::vector& macros, - absl::string_view description, - const ParserOptions& options) { - CEL_ASSIGN_OR_RETURN(auto verbose_parsed_expr, - EnrichedParse(expression, macros, description, options)); - return verbose_parsed_expr.parsed_expr(); -} - -absl::StatusOr EnrichedParse( - absl::string_view expression, const std::vector& macros, - absl::string_view description, const ParserOptions& options) { - CEL_ASSIGN_OR_RETURN(auto source, - cel::NewSource(expression, std::string(description))); - cel::MacroRegistry macro_registry; - CEL_RETURN_IF_ERROR(macro_registry.RegisterMacros(macros)); - return EnrichedParse(*source, macro_registry, options); -} - -absl::StatusOr EnrichedParse( - const cel::Source& source, const cel::MacroRegistry& registry, - const ParserOptions& options) { - ParsedExpr parsed_expr; - if (options.enable_pratt_parser) { - EnrichedSourceInfo enriched_source_info; - CEL_ASSIGN_OR_RETURN(std::unique_ptr ast, - cel::parser_internal::PrattParseImpl( - source, registry, options, - /*parse_issues=*/nullptr, &enriched_source_info)); - CEL_RETURN_IF_ERROR(cel::ast_internal::ExprToProto( - ast->root_expr(), parsed_expr.mutable_expr())); - CEL_RETURN_IF_ERROR(cel::ast_internal::SourceInfoToProto( - ast->source_info(), parsed_expr.mutable_source_info())); - return VerboseParsedExpr(std::move(parsed_expr), - std::move(enriched_source_info)); - } - CEL_ASSIGN_OR_RETURN(ParseResult parse_result, - ParseImpl(source, registry, options)); - CEL_RETURN_IF_ERROR(cel::ast_internal::ExprToProto( - parse_result.expr, parsed_expr.mutable_expr())); - - CEL_RETURN_IF_ERROR(cel::ast_internal::SourceInfoToProto( - parse_result.source_info, parsed_expr.mutable_source_info())); - return VerboseParsedExpr(std::move(parsed_expr), - std::move(parse_result.enriched_source_info)); -} - -absl::StatusOr Parse( +absl::StatusOr<::cel::expr::ParsedExpr> Parse( const cel::Source& source, const cel::MacroRegistry& registry, const ParserOptions& options) { CEL_ASSIGN_OR_RETURN(auto verbose_expr, @@ -1986,13 +123,15 @@ absl::StatusOr Parse( namespace cel { -// Creates a new parser builder. -// -// Intended for use with the Compiler class, most users should prefer the free -// functions above for independent parsing of expressions. std::unique_ptr NewParserBuilder(const ParserOptions& options) { - return std::make_unique( - options); +#ifdef EXCLUDE_CEL_ANTLR_PARSER + return parser_internal::NewPrattParserBuilder(options); +#else + if (options.enable_pratt_parser) { + return parser_internal::NewPrattParserBuilder(options); + } + return parser_internal::NewAntlrParserBuilder(options); +#endif } } // namespace cel diff --git a/parser/parser_test.cc b/parser/parser_test.cc index f7870b943..e2681608d 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -53,6 +53,14 @@ using ::cel::expr::Expr; using ::testing::HasSubstr; using ::testing::Not; +#ifdef EXCLUDE_CEL_ANTLR_PARSER +// Only test pratt parser. +#define TestedParserImpls testing::Values(true) +#else +// Test both pratt and antlr parsers. +#define TestedParserImpls testing::Values(true, false) +#endif + struct TestInfo { TestInfo(const std::string& I, const std::string& P, const std::string& E = "", const std::string& L = "", @@ -1802,17 +1810,6 @@ TEST_P(ExpressionImplTest, TsanOom) { .IgnoreError(); } -TEST_P(ExpressionTest, ErrorRecoveryLimits) { - ParserOptions options; - options.error_recovery_limit = 1; - auto result = Parse("......", "", options); - EXPECT_THAT(result, Not(IsOk())); - EXPECT_EQ(result.status().message(), - "ERROR: :1:1: Syntax error: More than 1 parse errors.\n | ......\n " - "| ^\nERROR: :1:2: Syntax error: no viable alternative at input " - "'..'\n | ......\n | .^"); -} - TEST_P(ExpressionImplTest, ExpressionSizeLimit) { options_.expression_size_codepoint_limit = 10; auto result = Parse("...............", "", options_); @@ -1836,21 +1833,6 @@ TEST_P(ExpressionImplTest, RecursionDepthLongArgList) { EXPECT_THAT(Parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]", "", options_), IsOk()); } -TEST(ExpressionTest, RecursionDepthExceeded_AntlrOnly) { - ParserOptions options; - options.enable_pratt_parser = false; - // AST visitor will recurse a variable amount depending on the terms used in - // the expression. This check occurs in the business logic converting the raw - // Antlr parse tree into an Expr. There is a separate check (via a custom - // listener) for AST depth while running the antlr generated parser. - options.max_recursion_depth = 6; - auto result = Parse("1 + 2 + 3 + 4 + 5 + 6 + 7", "", options); - - EXPECT_THAT(result, Not(IsOk())); - EXPECT_THAT(result.status().message(), - HasSubstr("Exceeded max recursion depth of 6 when parsing.")); -} - TEST_P(ExpressionImplTest, DisableQuotedIdentifiers) { options_.enable_quoted_identifiers = false; auto result = Parse("foo.`bar`", "", options_); @@ -2115,7 +2097,7 @@ INSTANTIATE_TEST_SUITE_P( " e^#7:Expr.Ident#\n" " )^#6:Expr.Call#\n" ")^#3:Expr.Call#"}), - testing::Bool()), + TestedParserImpls), VariadicLogicalOperatorsTestName); class ParserTest : public testing::TestWithParam { @@ -2223,23 +2205,22 @@ std::string ExpressionTestName( return name; } -INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionTest, - testing::Combine(testing::ValuesIn(test_cases), - testing::Bool()), - ExpressionTestName); - std::string ParserImplTestName(const testing::TestParamInfo& info) { return info.param ? "Pratt" : "Legacy"; } -INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionImplTest, testing::Bool(), - ParserImplTestName); +INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionTest, + testing::Combine(testing::ValuesIn(test_cases), + TestedParserImpls), + ExpressionTestName); -INSTANTIATE_TEST_SUITE_P(CelParserTest, NewParserBuilderTest, testing::Bool(), +INSTANTIATE_TEST_SUITE_P(CelParserTest, ExpressionImplTest, TestedParserImpls, ParserImplTestName); -INSTANTIATE_TEST_SUITE_P(CelParserTest, ParserTest, testing::Bool(), +INSTANTIATE_TEST_SUITE_P(CelParserTest, NewParserBuilderTest, TestedParserImpls, ParserImplTestName); +INSTANTIATE_TEST_SUITE_P(CelParserTest, ParserTest, TestedParserImpls, + ParserImplTestName); } // namespace } // namespace google::api::expr::parser From 1dcff093b3ca3d575f982e2ca154f9c751e06055 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 17 Aug 2026 11:46:40 -0700 Subject: [PATCH 181/350] Add GetCompilerOptions accessor to cel::Env. PiperOrigin-RevId: 966094202 --- env/env.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/env/env.h b/env/env.h index 9830b67d7..673ab19e4 100644 --- a/env/env.h +++ b/env/env.h @@ -59,6 +59,8 @@ class Env { void SetConfig(const Config& config) { config_ = config; } + CompilerOptions& GetCompilerOptions() { return compiler_options_; } + absl::StatusOr> NewCompilerBuilder(); // Shortcut for NewCompilerBuilder() followed by Build(). From 42ac7cc92b6e75d930817ece02b525f1186189a8 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 19 Aug 2026 15:29:33 -0700 Subject: [PATCH 182/350] Propagate unsafe bevhavior for arena owned messages. PiperOrigin-RevId: 967441637 --- common/value.cc | 14 ++++++ common/values/parsed_message_value.cc | 2 +- common/values/parsed_message_value.h | 11 +++-- eval/eval/select_step_test.cc | 63 ++++++++++++++------------- runtime/memory_safety_test.cc | 44 +++++++++++++++++++ 5 files changed, 99 insertions(+), 35 deletions(-) diff --git a/common/value.cc b/common/value.cc index fdb18d388..9ea8ec891 100644 --- a/common/value.cc +++ b/common/value.cc @@ -1512,6 +1512,13 @@ Value WrapFieldImpl( ABSL_DCHECK(!IsWellKnownMessageType(message->GetDescriptor())); const auto* reflection = message->GetReflection(); + if (ABSL_PREDICT_FALSE(reflection == nullptr)) { + // This only happens for special implementations of Message that + // should not normally be used with CEL. + return ErrorValue(absl::InvalidArgumentError( + absl::StrCat("failed to get reflection for message type: ", + message->GetDescriptor()->full_name()))); + } if (field->is_map()) { if (reflection->FieldSize(*message, field) == 0) { return MapValue(); @@ -1653,6 +1660,13 @@ Value WrapRepeatedFieldImpl( ABSL_DCHECK(arena != nullptr); const auto* reflection = message->GetReflection(); + if (ABSL_PREDICT_FALSE(reflection == nullptr)) { + // This only happens for special implementations of Message that + // should not normally be used with CEL. + return ErrorValue(absl::InvalidArgumentError( + absl::StrCat("failed to get reflection for message type: ", + message->GetDescriptor()->full_name()))); + } const int size = reflection->FieldSize(*message, field); if (ABSL_PREDICT_FALSE(index < 0 || index >= size)) { return ErrorValue(absl::InvalidArgumentError( diff --git a/common/values/parsed_message_value.cc b/common/values/parsed_message_value.cc index b0e881998..03d2d461d 100644 --- a/common/values/parsed_message_value.cc +++ b/common/values/parsed_message_value.cc @@ -382,7 +382,7 @@ absl::Status ParsedMessageValue::GetField( ABSL_DCHECK(arena != nullptr); ABSL_DCHECK(result != nullptr); - if (arena_ == nullptr) { + if (is_unsafe()) { *result = Value::WrapFieldUnsafe(unboxing_options, value_, field, descriptor_pool, message_factory, arena); } else { diff --git a/common/values/parsed_message_value.h b/common/values/parsed_message_value.h index 2e356d3e8..3cad912e7 100644 --- a/common/values/parsed_message_value.h +++ b/common/values/parsed_message_value.h @@ -192,7 +192,7 @@ class ParsedMessageValue final explicit ParsedMessageValue( const google::protobuf::Message* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND) - : value_(value), arena_(value->GetArena()) { + : value_(value), arena_(nullptr) { ABSL_DCHECK(value != nullptr); ABSL_DCHECK(!value_ || !IsWellKnownMessageType(value_->GetDescriptor())) << value_->GetTypeName() << " is a well known type"; @@ -210,9 +210,14 @@ class ParsedMessageValue final return absl::OkStatus(); } + bool is_unsafe() const { return arena_ == nullptr; } + const google::protobuf::Message* absl_nonnull value_; - // Arena that is attributed as owning the value. May be null to indicate that - // the value is managed externally. + + // The arena attributed as Owning this value. Null if the value is created by + // UnsafeParsedMessageValue() or derived from such a value. This is used to + // identify externally managed messages and propagating the unsafe field + // access behavior. google::protobuf::Arena* absl_nullable arena_; }; diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index 494def40f..f4dc3fcfb 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -92,7 +92,6 @@ class SelectStepTest : public testing::Test { // Helper method. Creates simple pipeline containing Select step and runs it. absl::StatusOr RunExpression(const CelValue target, absl::string_view field, bool test, - absl::string_view unknown_path, RunExpressionOptions options) { ExecutionPath path; @@ -118,6 +117,8 @@ class SelectStepTest : public testing::Test { runtime_options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; } + // Force the creation of a message factory at the env level. + static_cast(env_->MutableMessageFactory()); CelExpressionFlatImpl cel_expr( env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, env_->type_registry.GetComposedTypeProvider(), @@ -132,35 +133,20 @@ class SelectStepTest : public testing::Test { absl::string_view field, bool test, RunExpressionOptions options) { return RunExpression(CelProtoWrapper::CreateMessage(message, &arena_), - field, test, "", options); + field, test, options); } absl::StatusOr RunExpression(const TestMessage* message, absl::string_view field, bool test, - absl::string_view unknown_path, RunExpressionOptions options) { return RunExpression(CelProtoWrapper::CreateMessage(message, &arena_), - field, test, unknown_path, options); - } - - absl::StatusOr RunExpression(const TestMessage* message, - absl::string_view field, bool test, - RunExpressionOptions options) { - return RunExpression(message, field, test, "", options); + field, test, options); } absl::StatusOr RunExpression(const CelMap* map_value, absl::string_view field, bool test, - absl::string_view unknown_path, RunExpressionOptions options) { - return RunExpression(CelValue::CreateMap(map_value), field, test, - unknown_path, options); - } - - absl::StatusOr RunExpression(const CelMap* map_value, - absl::string_view field, bool test, - RunExpressionOptions options) { - return RunExpression(map_value, field, test, "", options); + return RunExpression(CelValue::CreateMap(map_value), field, test, options); } protected: @@ -189,8 +175,7 @@ TEST_P(SelectStepConformanceTest, SelectTargetNotStructOrMap) { ASSERT_OK_AND_ASSIGN( CelValue result, RunExpression(CelValue::CreateStringView("some_value"), "some_field", - /*test=*/false, - /*unknown_path=*/"", options)); + /*test=*/false, options)); ASSERT_TRUE(result.IsError()); EXPECT_THAT(*result.ErrorOrDie(), @@ -547,17 +532,36 @@ TEST_P(SelectStepConformanceTest, GlobalExtensionsMessageTest) { } TEST_P(SelectStepConformanceTest, GlobalExtensionsMessageUnsetTest) { - TestExtensions exts; + // Implementation details: + // The test environment is a dynamic descriptor pool with the same definition + // as the linked proto. + // Use a dynamic message with the expected factory and pool so we can expect + // the identity of the default message. + const google::protobuf::Descriptor* descriptor = + env_->descriptor_pool->FindMessageTypeByName( + TestExtensions::descriptor()->full_name()); + ASSERT_NE(descriptor, nullptr); + const google::protobuf::FieldDescriptor* field = + env_->descriptor_pool->FindExtensionByPrintableName( + descriptor, "google.api.expr.runtime.nested_ext"); + ASSERT_NE(field, nullptr); + ASSERT_TRUE(field->containing_type() == descriptor); + const auto* prototype = + env_->MutableMessageFactory()->GetPrototype(descriptor); + ASSERT_NE(prototype, nullptr); + const auto* msg_default = &prototype->GetReflection()->GetMessage( + *prototype, field, env_->MutableMessageFactory()); + RunExpressionOptions options; options.enable_unknowns = GetParam(); ASSERT_OK_AND_ASSIGN( CelValue result, - RunExpression(&exts, "google.api.expr.runtime.nested_ext", false, - options)); + RunExpression(CelProtoWrapper::CreateMessage(prototype, &arena_), + "google.api.expr.runtime.nested_ext", false, options)); ASSERT_TRUE(result.IsMessage()); - EXPECT_THAT(result.MessageOrDie(), Eq(&TestExtensions::default_instance())); + EXPECT_THAT(result.MessageOrDie(), Eq(msg_default)); } TEST_P(SelectStepConformanceTest, GlobalExtensionsWrapperTest) { @@ -653,18 +657,15 @@ TEST_P(SelectStepConformanceTest, NullMessageAccessor) { CelValue value = CelValue::CreateMessageWrapper( CelValue::MessageWrapper(&message, TrivialTypeInfo::GetInstance())); - ASSERT_OK_AND_ASSIGN(CelValue result, - RunExpression(value, "message_value", - /*test=*/false, - /*unknown_path=*/"", options)); + ASSERT_OK_AND_ASSIGN(CelValue result, RunExpression(value, "message_value", + /*test=*/false, options)); ASSERT_TRUE(result.IsError()); EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kNotFound)); // same for has ASSERT_OK_AND_ASSIGN(result, RunExpression(value, "message_value", - /*test=*/true, - /*unknown_path=*/"", options)); + /*test=*/true, options)); ASSERT_TRUE(result.IsError()); EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kNotFound)); diff --git a/runtime/memory_safety_test.cc b/runtime/memory_safety_test.cc index a60b4ce60..f2903d820 100644 --- a/runtime/memory_safety_test.cc +++ b/runtime/memory_safety_test.cc @@ -782,6 +782,50 @@ TEST_P(ViewTypesMemorySafetyTest, UnsafeWrappedMessageDifferentArena) { EXPECT_THAT(result_msg, IsSameInstance(&proto)); } +TEST_P(ViewTypesMemorySafetyTest, UnsafeWrappedMessageDifferentExplicitArena) { + // Arrange: create the runtime and expression. + ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, + ConfigureRuntimeImpl(false, EvaluationOptions())); + constexpr absl::string_view kProtoValue = R"pb( + child { payload { repeated_int32: [ 1, 2, 3 ] } } + payload { repeated_string: [ "foo", "bar", "baz" ] } + )pb"; + + ASSERT_OK_AND_ASSIGN( + ValidationResult validation, + GetCompiler().Compile( + "condition ? nested_test_all_types : NestedTestAllTypes{}")); + ASSERT_TRUE(validation.IsValid()) << validation.FormatError(); + ASSERT_OK_AND_ASSIGN(auto ast, validation.ReleaseAst()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr program, + runtime->CreateProgram(std::move(ast))); + + // Act: wrap the message and evaluate the expression. + // The unsafe version will alias the input message, so caller must ensure + // the input outlives the use of the `Value` rather than assuming it + // is managed by the evaluation arena. + google::protobuf::Arena arena; + google::protobuf::Arena other_arena; + auto* proto = google::protobuf::Arena::Create(&other_arena); + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(kProtoValue, proto)); + Activation activation; + activation.InsertOrAssignValue("condition", BoolValue(true)); + activation.InsertOrAssignValue( + "nested_test_all_types", + Value::WrapMessageUnsafe(proto, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), + &arena)); + ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); + + // Assert: the result is an alias of the input message. + ASSERT_TRUE(result.IsParsedMessage()); + const ParsedMessageValue& result_msg = result.GetParsedMessage(); + EXPECT_THAT(result_msg, + test::StructValueIs(ParsedProtoStructEquals(kProtoValue))); + EXPECT_EQ(result_msg->GetArena(), &other_arena); + EXPECT_THAT(result_msg, IsSameInstance(proto)); +} + TEST_P(ViewTypesMemorySafetyTest, UnsafeWrappedMessageFields) { // Arrange: create the runtime and expression. ASSERT_OK_AND_ASSIGN(std::unique_ptr runtime, From 00f138c2d105204de3b20eedb544b32a1932a270 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 19 Aug 2026 16:10:09 -0700 Subject: [PATCH 183/350] Update custom map `Find` and `Has` to treat errors as ErrorValue. Remove special case for empty map/list in wrap field. PiperOrigin-RevId: 967460976 --- common/value.cc | 7 +- common/values/custom_map_value.cc | 59 ++++-- common/values/custom_map_value.h | 15 ++ common/values/custom_map_value_test.cc | 256 +++++++++++++++++++++++++ eval/eval/BUILD | 1 + eval/eval/select_step_test.cc | 3 +- 6 files changed, 314 insertions(+), 27 deletions(-) diff --git a/common/value.cc b/common/value.cc index 9ea8ec891..e749a16c6 100644 --- a/common/value.cc +++ b/common/value.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include "google/protobuf/struct.pb.h" #include "absl/base/attributes.h" @@ -1520,9 +1521,6 @@ Value WrapFieldImpl( message->GetDescriptor()->full_name()))); } if (field->is_map()) { - if (reflection->FieldSize(*message, field) == 0) { - return MapValue(); - } if constexpr (Unsafe::value) { return UnsafeParsedMapFieldValue(message, field); } else { @@ -1531,9 +1529,6 @@ Value WrapFieldImpl( } } if (field->is_repeated()) { - if (reflection->FieldSize(*message, field) == 0) { - return ListValue(); - } if constexpr (Unsafe::value) { return UnsafeParsedRepeatedFieldValue(message, field); } else { diff --git a/common/values/custom_map_value.cc b/common/values/custom_map_value.cc index 495b64b7a..ecd04abfd 100644 --- a/common/values/custom_map_value.cc +++ b/common/values/custom_map_value.cc @@ -14,7 +14,9 @@ #include #include +#include #include +#include #include "absl/base/attributes.h" #include "absl/base/no_destructor.h" @@ -673,24 +675,34 @@ absl::StatusOr CustomMapValue::Find( return false; } - bool ok; if (dispatcher_ == nullptr) { CustomMapValueInterface::Content content = content_.To(); ABSL_DCHECK(content.interface != nullptr); - CEL_ASSIGN_OR_RETURN( - ok, content.interface->Find(key, descriptor_pool, message_factory, - arena, result)); - } else { - CEL_ASSIGN_OR_RETURN( - ok, dispatcher_->find(dispatcher_, content_, key, descriptor_pool, - message_factory, arena, result)); - } - if (ok) { + auto status_or_found = content.interface->Find( + key, descriptor_pool, message_factory, arena, result); + if (!status_or_found.ok()) { + *result = ErrorValue(std::move(status_or_found).status()); + return false; + } + if (!*status_or_found) { + *result = NullValue(); + return false; + } return true; } - *result = NullValue{}; - return false; + auto status_or_found = + dispatcher_->find(dispatcher_, content_, key, descriptor_pool, + message_factory, arena, result); + if (!status_or_found.ok()) { + *result = ErrorValue(std::move(status_or_found).status()); + return false; + } + if (!*status_or_found) { + *result = NullValue(); + return false; + } + return true; } absl::Status CustomMapValue::Has( @@ -721,19 +733,26 @@ absl::Status CustomMapValue::Has( *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); return absl::OkStatus(); } - bool has; if (dispatcher_ == nullptr) { CustomMapValueInterface::Content content = content_.To(); ABSL_DCHECK(content.interface != nullptr); - CEL_ASSIGN_OR_RETURN(has, content.interface->Has(key, descriptor_pool, - message_factory, arena)); - } else { - CEL_ASSIGN_OR_RETURN( - has, dispatcher_->has(dispatcher_, content_, key, descriptor_pool, - message_factory, arena)); + auto status_or_has = + content.interface->Has(key, descriptor_pool, message_factory, arena); + if (!status_or_has.ok()) { + *result = ErrorValue(std::move(status_or_has).status()); + return absl::OkStatus(); + } + *result = BoolValue(*status_or_has); + return absl::OkStatus(); + } + auto status_or_has = dispatcher_->has( + dispatcher_, content_, key, descriptor_pool, message_factory, arena); + if (!status_or_has.ok()) { + *result = ErrorValue(std::move(status_or_has).status()); + return absl::OkStatus(); } - *result = BoolValue(has); + *result = BoolValue(*status_or_has); return absl::OkStatus(); } diff --git a/common/values/custom_map_value.h b/common/values/custom_map_value.h index ca6e1e025..5d2a1dfb5 100644 --- a/common/values/custom_map_value.h +++ b/common/values/custom_map_value.h @@ -54,6 +54,12 @@ class CustomMapValueInterfaceKeysIterator; class CustomMapValue; using CustomMapValueContent = CustomValueContent; +// Dispatch table for `CustomMapValue`. +// +// See the documentation for `CustomMapValueInterface` for more details on +// composite functions. +// +// See documentation for `UnsafeCustomMapValue` on how to use this class. struct CustomMapValueDispatcher { using GetTypeId = NativeTypeId (*)(const CustomMapValueDispatcher* absl_nonnull dispatcher, @@ -247,12 +253,21 @@ class CustomMapValueInterface { virtual CustomMapValue Clone(google::protobuf::Arena* absl_nonnull arena) const = 0; + // Tests whether the map contains the given key. If it does, the value + // associated with the key is written to `result` and the function returns + // true. Otherwise, the function returns false and `result` is set to + // `NullValue`. + // + // A non-ok status is converted to an ErrorValue (e.g. wrong key type). virtual absl::StatusOr Find( const Value& key, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const = 0; + // Whether the map has the given key. + // + // A non-ok status is converted to an ErrorValue (e.g. wrong key type). virtual absl::StatusOr Has( const Value& key, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, diff --git a/common/values/custom_map_value_test.cc b/common/values/custom_map_value_test.cc index d19bca373..11c46d4cf 100644 --- a/common/values/custom_map_value_test.cc +++ b/common/values/custom_map_value_test.cc @@ -139,6 +139,9 @@ class CustomMapValueInterfaceTest final : public CustomMapValueInterface { *result = IntValue(1); return true; } + if (*string_key == "error") { + return absl::InvalidArgumentError("custom error"); + } } return false; } @@ -155,6 +158,9 @@ class CustomMapValueInterfaceTest final : public CustomMapValueInterface { if (*string_key == "bar") { return true; } + if (*string_key == "error") { + return absl::InvalidArgumentError("custom error"); + } } return false; } @@ -254,6 +260,9 @@ class CustomMapValueTest : public common_internal::ValueTest<> { *result = IntValue(1); return true; } + if (*string_key == "error") { + return absl::InvalidArgumentError("custom error"); + } } return false; }, @@ -269,6 +278,9 @@ class CustomMapValueTest : public common_internal::ValueTest<> { if (*string_key == "bar") { return true; } + if (*string_key == "error") { + return absl::InvalidArgumentError("custom error"); + } } return false; }, @@ -496,6 +508,150 @@ TEST_F(CustomMapValueTest, Interface_Find) { IsOkAndHolds(Eq(std::nullopt))); } +TEST_F(CustomMapValueTest, Dispatcher_Find_Error) { + CustomMapValue map = MakeDispatcher(); + Value result; + ASSERT_THAT(map.Find(StringValue("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + ASSERT_THAT(map.Get(StringValue("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + EXPECT_THAT(map.Get(StringValue("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs( + absl::StatusCode::kInvalidArgument, "custom error")))); + EXPECT_THAT(map.Find(StringValue("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(CustomMapValueTest, Interface_Find_Error) { + CustomMapValue map = MakeInterface(); + Value result; + ASSERT_THAT(map.Find(StringValue("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + ASSERT_THAT(map.Get(StringValue("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + EXPECT_THAT(map.Get(StringValue("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs( + absl::StatusCode::kInvalidArgument, "custom error")))); + EXPECT_THAT(map.Find(StringValue("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Find_InvalidKeyType) { + CustomMapValue map = MakeDispatcher(); + Value result; + ASSERT_THAT(map.Find(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + ASSERT_THAT(map.Get(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + EXPECT_THAT( + map.Get(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); +} + +TEST_F(CustomMapValueTest, Interface_Find_InvalidKeyType) { + CustomMapValue map = MakeInterface(); + Value result; + ASSERT_THAT(map.Find(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + ASSERT_THAT(map.Get(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + EXPECT_THAT( + map.Get(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Find_SpecialKeys) { + CustomMapValue map = MakeDispatcher(); + Value result; + ErrorValue error_key(absl::CancelledError("cancelled")); + ASSERT_THAT(map.Find(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + ASSERT_THAT(map.Get(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + EXPECT_THAT(map.Get(error_key, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled")))); + + UnknownValue unknown_key; + ASSERT_THAT(map.Find(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOkAndHolds(false)); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_THAT(map.Get(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_OK_AND_ASSIGN(auto get_result, map.Get(unknown_key, descriptor_pool(), + message_factory(), arena())); + EXPECT_TRUE(get_result.IsUnknown()); +} + +TEST_F(CustomMapValueTest, Interface_Find_SpecialKeys) { + CustomMapValue map = MakeInterface(); + Value result; + ErrorValue error_key(absl::CancelledError("cancelled")); + ASSERT_THAT(map.Find(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOkAndHolds(false)); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + ASSERT_THAT(map.Get(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + EXPECT_THAT(map.Get(error_key, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled")))); + + UnknownValue unknown_key; + ASSERT_THAT(map.Find(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOkAndHolds(false)); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_THAT(map.Get(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_OK_AND_ASSIGN(auto get_result, map.Get(unknown_key, descriptor_pool(), + message_factory(), arena())); + EXPECT_TRUE(get_result.IsUnknown()); +} + TEST_F(CustomMapValueTest, Dispatcher_Has) { CustomMapValue map = MakeDispatcher(); ASSERT_THAT(map.Has(StringValue("foo"), descriptor_pool(), message_factory(), @@ -522,6 +678,106 @@ TEST_F(CustomMapValueTest, Interface_Has) { IsOkAndHolds(BoolValueIs(false))); } +TEST_F(CustomMapValueTest, Dispatcher_Has_Error) { + CustomMapValue map = MakeDispatcher(); + Value result; + ASSERT_THAT(map.Has(StringValue("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + EXPECT_THAT(map.Has(StringValue("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs( + absl::StatusCode::kInvalidArgument, "custom error")))); +} + +TEST_F(CustomMapValueTest, Interface_Has_Error) { + CustomMapValue map = MakeInterface(); + Value result; + ASSERT_THAT(map.Has(StringValue("error"), descriptor_pool(), + message_factory(), arena(), &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, + "custom error"))); + EXPECT_THAT(map.Has(StringValue("error"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs( + absl::StatusCode::kInvalidArgument, "custom error")))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Has_InvalidKeyType) { + CustomMapValue map = MakeDispatcher(); + Value result; + ASSERT_THAT(map.Has(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + EXPECT_THAT( + map.Has(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); +} + +TEST_F(CustomMapValueTest, Interface_Has_InvalidKeyType) { + CustomMapValue map = MakeInterface(); + Value result; + ASSERT_THAT(map.Has(DoubleValue(1.0), descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_THAT(result, + ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); + EXPECT_THAT( + map.Has(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); +} + +TEST_F(CustomMapValueTest, Dispatcher_Has_SpecialKeys) { + CustomMapValue map = MakeDispatcher(); + Value result; + ErrorValue error_key(absl::CancelledError("cancelled")); + ASSERT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + EXPECT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled")))); + + UnknownValue unknown_key; + ASSERT_THAT(map.Has(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_OK_AND_ASSIGN(auto has_result, map.Has(unknown_key, descriptor_pool(), + message_factory(), arena())); + EXPECT_TRUE(has_result.IsUnknown()); +} + +TEST_F(CustomMapValueTest, Interface_Has_SpecialKeys) { + CustomMapValue map = MakeInterface(); + Value result; + ErrorValue error_key(absl::CancelledError("cancelled")); + ASSERT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena(), + &result), + IsOk()); + EXPECT_THAT(result, ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled"))); + EXPECT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(ErrorValueIs( + StatusIs(absl::StatusCode::kCancelled, "cancelled")))); + + UnknownValue unknown_key; + ASSERT_THAT(map.Has(unknown_key, descriptor_pool(), message_factory(), + arena(), &result), + IsOk()); + EXPECT_TRUE(result.IsUnknown()); + ASSERT_OK_AND_ASSIGN(auto has_result, map.Has(unknown_key, descriptor_pool(), + message_factory(), arena())); + EXPECT_TRUE(has_result.IsUnknown()); +} + TEST_F(CustomMapValueTest, Dispatcher_ForEach) { std::vector> entries; EXPECT_THAT( diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 507ff6f0e..329ee71f4 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -828,6 +828,7 @@ cc_test( "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index f4dc3fcfb..92cda2fbe 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -13,6 +13,7 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" @@ -311,7 +312,7 @@ TEST_F(SelectStepTest, MapPresenseIsErrorTest) { CelProtoWrapper::CreateMessage(&message, &arena_)); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); - EXPECT_TRUE(result.IsError()); + ASSERT_TRUE(result.IsError()); EXPECT_EQ(result.ErrorOrDie()->code(), absl::StatusCode::kInvalidArgument); } From a3acb2ffad180887e6c7a72af353cbced7a0bad7 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 20 Aug 2026 10:40:16 -0700 Subject: [PATCH 184/350] Fix bug in ::Clone impl for repeated fields This would crash if the repeated field was not a message. This wasn't likely to be triggered but possible for interop where cel::Runtime API was used with some (adapted) legacy CelFunction implementations. Add additional coverage. PiperOrigin-RevId: 967916085 --- common/values/parsed_json_list_value_test.cc | 40 +++++++++- common/values/parsed_json_map_value_test.cc | 51 +++++++++++++ common/values/parsed_map_field_value.cc | 3 +- common/values/parsed_map_field_value_test.cc | 42 +++++++++++ common/values/parsed_repeated_field_value.cc | 72 ++++++++++++++++-- .../parsed_repeated_field_value_test.cc | 74 ++++++++++++++++++- 6 files changed, 270 insertions(+), 12 deletions(-) diff --git a/common/values/parsed_json_list_value_test.cc b/common/values/parsed_json_list_value_test.cc index a7ce17410..5fcff20bf 100644 --- a/common/values/parsed_json_list_value_test.cc +++ b/common/values/parsed_json_list_value_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -21,14 +22,15 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/type.h" #include "common/value.h" #include "common/value_kind.h" #include "common/value_testing.h" +#include "internal/parse_text_proto.h" #include "internal/testing.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/arena.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" namespace cel { @@ -285,5 +287,41 @@ TEST_F(ParsedJsonListValueTest, Contains_Dynamic) { IsOkAndHolds(BoolValueIs(true))); } +TEST_F(ParsedJsonListValueTest, CloneDefault) { + ParsedJsonListValue value; + EXPECT_FALSE(value.Clone(arena())); +} + +TEST_F(ParsedJsonListValueTest, CloneSameArena) { + ParsedJsonListValue value(DynamicParseTextProto( + R"pb(values { null_value: NULL_VALUE } + values { bool_value: true })pb"), + arena()); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + +TEST_F(ParsedJsonListValueTest, CloneDifferentArena) { + google::protobuf::Arena other_arena; + ParsedJsonListValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, + R"pb(values { null_value: NULL_VALUE } + values { bool_value: true })pb", + descriptor_pool(), message_factory()), + &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(0, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(IsNullValue())); + EXPECT_THAT(cloned.Get(1, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + } // namespace } // namespace cel diff --git a/common/values/parsed_json_map_value_test.cc b/common/values/parsed_json_map_value_test.cc index acc29deb2..67bddcf8e 100644 --- a/common/values/parsed_json_map_value_test.cc +++ b/common/values/parsed_json_map_value_test.cc @@ -336,5 +336,56 @@ TEST_F(ParsedJsonMapValueTest, NewIterator2) { IsOkAndHolds(Eq(std::nullopt))); } +TEST_F(ParsedJsonMapValueTest, CloneDefault) { + ParsedJsonMapValue value; + EXPECT_FALSE(value.Clone(arena())); +} + +TEST_F(ParsedJsonMapValueTest, CloneSameArena) { + ParsedJsonMapValue value(DynamicParseTextProto(R"pb( + fields { + key: "foo" + value: { null_value: NULL_VALUE } + } + fields { + key: "bar" + value: { bool_value: true } + })pb"), + arena()); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + +TEST_F(ParsedJsonMapValueTest, CloneDifferentArena) { + google::protobuf::Arena other_arena; + ParsedJsonMapValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, + R"pb( + fields { + key: "foo" + value: { null_value: NULL_VALUE } + } + fields { + key: "bar" + value: { bool_value: true } + })pb", + descriptor_pool(), message_factory()), + &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(StringValue("foo"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(IsNullValue())); + EXPECT_THAT(cloned.Get(StringValue("bar"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + } // namespace } // namespace cel diff --git a/common/values/parsed_map_field_value.cc b/common/values/parsed_map_field_value.cc index 92787a8af..0b2b49a8d 100644 --- a/common/values/parsed_map_field_value.cc +++ b/common/values/parsed_map_field_value.cc @@ -157,9 +157,8 @@ bool ParsedMapFieldValue::IsZeroValue() const { return IsEmpty(); } ParsedMapFieldValue ParsedMapFieldValue::Clone( google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); - ABSL_DCHECK(*this); - if (ABSL_PREDICT_FALSE(field_ == nullptr)) { + if (ABSL_PREDICT_FALSE(!*this)) { return ParsedMapFieldValue(); } if (arena_ == arena) { diff --git a/common/values/parsed_map_field_value_test.cc b/common/values/parsed_map_field_value_test.cc index f55eb65ed..e5f79dc39 100644 --- a/common/values/parsed_map_field_value_test.cc +++ b/common/values/parsed_map_field_value_test.cc @@ -640,5 +640,47 @@ TEST_F(ParsedMapFieldValueTest, NewIterator2) { IsOkAndHolds(Eq(std::nullopt))); } +TEST_F(ParsedMapFieldValueTest, CloneDefault) { + ParsedMapFieldValue value; + EXPECT_FALSE(value.Clone(arena())); +} + +TEST_F(ParsedMapFieldValueTest, CloneSameArena) { + ParsedMapFieldValue value( + DynamicParseTextProto(R"pb( + map_string_string { key: "foo" value: "bar" } + map_string_string { key: "bar" value: "foo" } + )pb"), + DynamicGetField("map_string_string"), arena()); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + +TEST_F(ParsedMapFieldValueTest, CloneDifferentArena) { + google::protobuf::Arena other_arena; + ParsedMapFieldValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, + R"pb( + map_string_string { key: "foo" value: "bar" } + map_string_string { key: "bar" value: "foo" } + )pb", + descriptor_pool(), message_factory()), + DynamicGetField("map_string_string"), &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(StringValue("foo"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(StringValueIs("bar"))); + EXPECT_THAT(cloned.Get(StringValue("bar"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(StringValueIs("foo"))); +} + } // namespace } // namespace cel diff --git a/common/values/parsed_repeated_field_value.cc b/common/values/parsed_repeated_field_value.cc index b990d3965..db9a810ff 100644 --- a/common/values/parsed_repeated_field_value.cc +++ b/common/values/parsed_repeated_field_value.cc @@ -15,6 +15,7 @@ #include "common/values/parsed_repeated_field_value.h" #include +#include #include #include #include @@ -147,24 +148,79 @@ absl::Status ParsedRepeatedFieldValue::Equal( bool ParsedRepeatedFieldValue::IsZeroValue() const { return IsEmpty(); } +namespace { + +template +void CopyRepeatedFieldImpl(const google::protobuf::Reflection* absl_nonnull reflection, + const google::protobuf::Message& src_message, + google::protobuf::Message* absl_nonnull dst_message, + const google::protobuf::FieldDescriptor* absl_nonnull field) { + auto src_field = reflection->GetRepeatedFieldRef(src_message, field); + auto dst_field = + reflection->GetMutableRepeatedFieldRef(dst_message, field); + dst_field.CopyFrom(src_field); +} + +void CopyRepeatedField(const google::protobuf::Reflection* absl_nonnull reflection, + const google::protobuf::Message& src_message, + google::protobuf::Message* absl_nonnull dst_message, + const google::protobuf::FieldDescriptor* absl_nonnull field) { + switch (field->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: + CopyRepeatedFieldImpl(reflection, src_message, dst_message, + field); + break; + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + CopyRepeatedFieldImpl(reflection, src_message, + dst_message, field); + break; + } +} + +} // namespace + ParsedRepeatedFieldValue ParsedRepeatedFieldValue::Clone( google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); - ABSL_DCHECK(*this); - if (ABSL_PREDICT_FALSE(field_ == nullptr)) { + if (ABSL_PREDICT_FALSE(!*this)) { return ParsedRepeatedFieldValue(); } if (arena_ == arena) { return *this; } - auto field = message_->GetReflection()->GetRepeatedFieldRef( - *message_, field_); auto* cloned_message = message_->New(arena); - auto cloned_field = - cloned_message->GetReflection() - ->GetMutableRepeatedFieldRef(cloned_message, field_); - cloned_field.CopyFrom(field); + CopyRepeatedField(GetReflection(), *message_, cloned_message, field_); return ParsedRepeatedFieldValue(cloned_message, field_, arena); } diff --git a/common/values/parsed_repeated_field_value_test.cc b/common/values/parsed_repeated_field_value_test.cc index 6941de0e9..a3a80456a 100644 --- a/common/values/parsed_repeated_field_value_test.cc +++ b/common/values/parsed_repeated_field_value_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -22,14 +23,15 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/type.h" #include "common/value.h" #include "common/value_kind.h" #include "common/value_testing.h" +#include "internal/parse_text_proto.h" #include "internal/testing.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" +#include "google/protobuf/arena.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" namespace cel { @@ -45,6 +47,7 @@ using ::cel::test::DurationValueIs; using ::cel::test::ErrorValueIs; using ::cel::test::IntValueIs; using ::cel::test::IsNullValue; +using ::cel::test::StringValueIs; using ::cel::test::UintValueIs; using ::testing::_; using ::testing::ElementsAre; @@ -446,5 +449,74 @@ TEST_F(ParsedRepeatedFieldValueTest, Contains) { IsOkAndHolds(BoolValueIs(false))); } +TEST_F(ParsedRepeatedFieldValueTest, CloneDefault) { + ParsedRepeatedFieldValue value; + EXPECT_FALSE(value.Clone(arena())); +} + +TEST_F(ParsedRepeatedFieldValueTest, CloneSameArena) { + ParsedRepeatedFieldValue value( + DynamicParseTextProto(R"pb(repeated_int64: 1 + repeated_int64: 2)pb"), + DynamicGetField("repeated_int64"), arena()); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); +} + +TEST_F(ParsedRepeatedFieldValueTest, CloneDifferentArenaInt64) { + google::protobuf::Arena other_arena; + ParsedRepeatedFieldValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, R"pb(repeated_int64: 1 repeated_int64: 2)pb", + descriptor_pool(), message_factory()), + DynamicGetField("repeated_int64"), &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(0, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(IntValueIs(1))); + EXPECT_THAT(cloned.Get(1, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(IntValueIs(2))); +} + +TEST_F(ParsedRepeatedFieldValueTest, CloneDifferentArenaString) { + google::protobuf::Arena other_arena; + ParsedRepeatedFieldValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, R"pb(repeated_string: "foo" repeated_string: "bar")pb", + descriptor_pool(), message_factory()), + DynamicGetField("repeated_string"), &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); + EXPECT_THAT(cloned.Get(0, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(StringValueIs("foo"))); + EXPECT_THAT(cloned.Get(1, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(StringValueIs("bar"))); +} + +TEST_F(ParsedRepeatedFieldValueTest, CloneDifferentArenaMessage) { + google::protobuf::Arena other_arena; + ParsedRepeatedFieldValue value( + ::cel::internal::DynamicParseTextProto( + &other_arena, + R"pb(repeated_nested_message: { bb: 1 } + repeated_nested_message: { bb: 2 })pb", + descriptor_pool(), message_factory()), + DynamicGetField("repeated_nested_message"), + &other_arena); + auto cloned = value.Clone(arena()); + EXPECT_THAT( + cloned.Equal(value, descriptor_pool(), message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); + EXPECT_EQ(cloned.Size(), 2); +} + } // namespace } // namespace cel From 87f68d89acde44f5d88fd8f15a2c8cf7c8864af4 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 20 Aug 2026 13:48:49 -0700 Subject: [PATCH 185/350] Fix bug in map field to JSON conversion. Any packing logic for a field-backed map would silently fail. PiperOrigin-RevId: 968032735 --- common/values/parsed_map_field_value.cc | 2 +- common/values/parsed_map_field_value_test.cc | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/common/values/parsed_map_field_value.cc b/common/values/parsed_map_field_value.cc index 0b2b49a8d..8dd7c3d02 100644 --- a/common/values/parsed_map_field_value.cc +++ b/common/values/parsed_map_field_value.cc @@ -69,7 +69,7 @@ absl::Status ParsedMapFieldValue::SerializeTo( google::protobuf::Value message; CEL_RETURN_IF_ERROR(internal::MessageFieldToJson( *message_, field_, descriptor_pool, message_factory, &message)); - if (!message.list_value().SerializePartialToZeroCopyStream(output)) { + if (!message.struct_value().SerializePartialToZeroCopyStream(output)) { return absl::UnknownError("failed to serialize google.protobuf.Struct"); } return absl::OkStatus(); diff --git a/common/values/parsed_map_field_value_test.cc b/common/values/parsed_map_field_value_test.cc index e5f79dc39..c1a968646 100644 --- a/common/values/parsed_map_field_value_test.cc +++ b/common/values/parsed_map_field_value_test.cc @@ -102,7 +102,7 @@ TEST_F(ParsedMapFieldValueTest, IsZeroValue) { EXPECT_TRUE(value.IsZeroValue()); } -TEST_F(ParsedMapFieldValueTest, SerializeTo) { +TEST_F(ParsedMapFieldValueTest, SerializeToEmpty) { ParsedMapFieldValue value( DynamicParseTextProto(R"pb()pb"), DynamicGetField("map_int64_int64"), arena()); @@ -112,6 +112,23 @@ TEST_F(ParsedMapFieldValueTest, SerializeTo) { EXPECT_THAT(std::move(output).Consume(), IsEmpty()); } +TEST_F(ParsedMapFieldValueTest, SerializeTo) { + ParsedMapFieldValue value( + DynamicParseTextProto( + R"pb(map_string_string { key: "foo" value: "bar" })pb"), + DynamicGetField("map_string_string"), arena()); + google::protobuf::io::CordOutputStream output; + EXPECT_THAT(value.SerializeTo(descriptor_pool(), message_factory(), &output), + IsOk()); + google::protobuf::Struct actual; + EXPECT_TRUE(actual.ParsePartialFromCord(std::move(output).Consume())); + EXPECT_THAT(actual, EqualsTextProto( + R"pb(fields { + key: "foo" + value { string_value: "bar" } + })pb")); +} + TEST_F(ParsedMapFieldValueTest, ConvertToJson) { auto json = DynamicParseTextProto(R"pb()pb"); ParsedMapFieldValue value( From 5053f465b5826b934d4ee63f784a19f4cfb3edcb Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 20 Aug 2026 15:47:44 -0700 Subject: [PATCH 186/350] Drop bazel dependency on flatbuffers. This was only used for an optional extension. Downstreamed the implementation to the only known user. PiperOrigin-RevId: 968101573 --- MODULE.bazel | 5 ----- tools/BUILD | 35 --------------------------------- tools/flatbuffers_backed_impl.h | 4 ++++ tools/testdata/BUILD | 18 ----------------- 4 files changed, 4 insertions(+), 58 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b013fb86e..82667a5b6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -57,11 +57,6 @@ bazel_dep( version = "2025-11-05.bcr.1", repo_name = "com_googlesource_code_re2", ) -bazel_dep( - name = "flatbuffers", - version = "25.9.23", - repo_name = "com_github_google_flatbuffers", -) bazel_dep( name = "cel-spec", version = "0.25.2", diff --git a/tools/BUILD b/tools/BUILD index af006a67b..8b6ba13d9 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -74,41 +74,6 @@ cc_test( ], ) -cc_library( - name = "flatbuffers_backed_impl", - srcs = [ - "flatbuffers_backed_impl.cc", - ], - hdrs = [ - "flatbuffers_backed_impl.h", - ], - deps = [ - "//eval/public:cel_value", - "@com_github_google_flatbuffers//:flatbuffers", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - ], -) - -cc_test( - name = "flatbuffers_backed_impl_test", - size = "small", - srcs = [ - "flatbuffers_backed_impl_test.cc", - ], - data = [ - "//tools/testdata:flatbuffers_reflection_out", - ], - deps = [ - ":flatbuffers_backed_impl", - "//internal:status_macros", - "//internal:testing", - "@com_github_google_flatbuffers//:flatbuffers", - ], -) - cc_library( name = "navigable_ast", srcs = ["navigable_ast.cc"], diff --git a/tools/flatbuffers_backed_impl.h b/tools/flatbuffers_backed_impl.h index 7051ef5d5..bf02436ad 100644 --- a/tools/flatbuffers_backed_impl.h +++ b/tools/flatbuffers_backed_impl.h @@ -9,6 +9,10 @@ namespace api { namespace expr { namespace runtime { +// Reference implementation of CelMap backed by a flatbuffer table. +// +// To avoid taking a dependency on flatbuffers, this is not included in the +// bazel module or built by public CI. class FlatBuffersMapImpl : public CelMap { public: FlatBuffersMapImpl(const flatbuffers::Table& table, diff --git a/tools/testdata/BUILD b/tools/testdata/BUILD index c88c9c478..f9d8ea4b9 100644 --- a/tools/testdata/BUILD +++ b/tools/testdata/BUILD @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public") load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") @@ -21,14 +20,6 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) -flatbuffer_library_public( - name = "flatbuffers_test", - srcs = ["flatbuffers.fbs"], - outs = ["flatbuffers_generated.h"], - language_flag = "-c", - reflection_name = "flatbuffers_reflection", -) - filegroup( name = "coverage_testdata", srcs = [ @@ -37,15 +28,6 @@ filegroup( ], ) -cc_library( - name = "flatbuffers_test_cc", - srcs = [":flatbuffers_test"], - hdrs = [":flatbuffers_test"], - features = ["-parse_headers"], - linkstatic = True, - deps = ["@com_github_google_flatbuffers//:runtime_cc"], -) - proto_library( name = "test_policy_proto", srcs = ["test_policy.proto"], From ba959b40c23be5518bef3c298a00219b20ae4141 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 24 Aug 2026 11:16:27 -0700 Subject: [PATCH 187/350] Add option to enable/disable repeated unary op folding in parser. PiperOrigin-RevId: 969965681 --- parser/internal/BUILD | 1 + parser/internal/antlr_parser.cc | 54 ++++++++++++++++++--------- parser/internal/antlr_parser_test.cc | 16 ++++++++ parser/internal/pratt_parser_test.cc | 10 +---- parser/internal/pratt_parser_worker.h | 17 ++++++++- parser/options.h | 16 ++++++++ parser/parser_test.cc | 6 +-- 7 files changed, 89 insertions(+), 31 deletions(-) diff --git a/parser/internal/BUILD b/parser/internal/BUILD index cebb5c329..d679e2853 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -117,6 +117,7 @@ cc_test( deps = [ ":antlr_parser", "//common:ast", + "//common:navigable_ast", "//common:source", "//internal:status_macros", "//internal:testing", diff --git a/parser/internal/antlr_parser.cc b/parser/internal/antlr_parser.cc index 6116d05a3..cb9e63258 100644 --- a/parser/internal/antlr_parser.cc +++ b/parser/internal/antlr_parser.cc @@ -615,7 +615,8 @@ class ParserVisitor final : public CelBaseVisitor, bool add_macro_calls = false, bool enable_optional_syntax = false, bool enable_quoted_identifiers = false, - bool enable_variadic_logical_operators = false) + bool enable_variadic_logical_operators = false, + bool fold_unary_operators = false) : source_(source), factory_(source_, max_expression_node_count), macro_registry_(macro_registry), @@ -624,7 +625,8 @@ class ParserVisitor final : public CelBaseVisitor, add_macro_calls_(add_macro_calls), enable_optional_syntax_(enable_optional_syntax), enable_quoted_identifiers_(enable_quoted_identifiers), - enable_variadic_logical_operators_(enable_variadic_logical_operators) {} + enable_variadic_logical_operators_(enable_variadic_logical_operators), + fold_unary_operators_(fold_unary_operators) {} ~ParserVisitor() override = default; @@ -700,6 +702,9 @@ class ParserVisitor final : public CelBaseVisitor, const Expr& e); std::string NormalizeIdentifier(CelParser::EscapeIdentContext* ctx); + std::any VisitUnaryOps(const std::vector& ops, + CelParser::MemberContext* member, + absl::string_view op_name); // Attempt to unnest parse context. // // Walk the parse tree to the first complex term to reduce recursive depth in @@ -716,6 +721,7 @@ class ParserVisitor final : public CelBaseVisitor, const bool enable_optional_syntax_; const bool enable_quoted_identifiers_; const bool enable_variadic_logical_operators_; + const bool fold_unary_operators_; }; template >")); } -std::any ParserVisitor::visitLogicalNot(CelParser::LogicalNotContext* ctx) { - if (ctx->ops.size() % 2 == 0) { - return visit(ctx->member()); +std::any ParserVisitor::VisitUnaryOps(const std::vector& ops, + CelParser::MemberContext* member, + absl::string_view op_name) { + if (fold_unary_operators_) { + if (ops.size() % 2 == 0) { + return visit(member); + } + int64_t op_id = factory_.NextId(SourceRangeFromToken(ops[0])); + auto target = ExprFromAny(visit(member)); + return ExprToAny(GlobalCallOrMacro(op_id, op_name, std::move(target))); } - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->ops[0])); - auto target = ExprFromAny(visit(ctx->member())); - return ExprToAny( - GlobalCallOrMacro(op_id, CelOperator::LOGICAL_NOT, std::move(target))); + + std::vector op_ids; + op_ids.reserve(ops.size()); + for (const auto* op : ops) { + op_ids.push_back(factory_.NextId(SourceRangeFromToken(op))); + } + + auto target = ExprFromAny(visit(member)); + for (int i = static_cast(op_ids.size()) - 1; i >= 0; --i) { + target = GlobalCallOrMacro(op_ids[i], op_name, std::move(target)); + } + return ExprToAny(std::move(target)); +} + +std::any ParserVisitor::visitLogicalNot(CelParser::LogicalNotContext* ctx) { + return VisitUnaryOps(ctx->ops, ctx->member(), CelOperator::LOGICAL_NOT); } std::any ParserVisitor::visitNegate(CelParser::NegateContext* ctx) { - if (ctx->ops.size() % 2 == 0) { - return visit(ctx->member()); - } - int64_t op_id = factory_.NextId(SourceRangeFromToken(ctx->ops[0])); - auto target = ExprFromAny(visit(ctx->member())); - return ExprToAny( - GlobalCallOrMacro(op_id, CelOperator::NEGATE, std::move(target))); + return VisitUnaryOps(ctx->ops, ctx->member(), CelOperator::NEGATE); } std::string ParserVisitor::NormalizeIdentifier( @@ -1684,7 +1703,8 @@ absl::StatusOr> AntlrParseImpl( source, options.max_recursion_depth, options.expression_node_limit, registry, options.add_macro_calls, options.enable_optional_syntax, options.enable_quoted_identifiers, - options.enable_variadic_logical_operators); + options.enable_variadic_logical_operators, + options.fold_unary_operators); lexer.removeErrorListeners(); parser.removeErrorListeners(); diff --git a/parser/internal/antlr_parser_test.cc b/parser/internal/antlr_parser_test.cc index d1a99227a..4bdafc42b 100644 --- a/parser/internal/antlr_parser_test.cc +++ b/parser/internal/antlr_parser_test.cc @@ -23,6 +23,7 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "common/ast.h" +#include "common/navigable_ast.h" #include "common/source.h" #include "internal/status_macros.h" #include "internal/testing.h" @@ -72,5 +73,20 @@ TEST(AntlrParserTest, RecursionDepthExceeded) { HasSubstr("Exceeded max recursion depth of 6 when parsing.")); } +TEST(AntlrParserTest, UnaryOperatorsUnfoldedOption) { + ParserOptions options; + options.fold_unary_operators = false; + + ASSERT_OK_AND_ASSIGN(auto ast, Parse("---a", "", options)); + auto nav_ast = cel::NavigableAst::Build(ast->root_expr()); + EXPECT_EQ(nav_ast.Root().height(), 4); + + for (const auto& node : nav_ast.Root().DescendantsPostorder()) { + if (node.node_kind() == cel::NodeKind::kCall) { + EXPECT_EQ(node.expr()->call_expr().function(), "-_"); + } + } +} + } // namespace } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 8ebe5123e..116ae607b 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -434,9 +434,7 @@ std::vector GetParserTestCases() { TestCase{ .source = "- -1", .expected_ast = R"( - -_( - -1^#2:int64# - )^#1:Expr.Call# + 1^#3:int64# )", }, TestCase{ @@ -454,11 +452,7 @@ std::vector GetParserTestCases() { .source = "---a", .expected_ast = R"( -_( - -_( - -_( - a^#4:Expr.Ident# - )^#3:Expr.Call# - )^#2:Expr.Call# + a^#4:Expr.Ident# )^#1:Expr.Call# )", }, diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index bbd719e34..dc17f27d6 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -544,7 +544,15 @@ ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { ExprNode operand; if (!ops.empty() && ops.back().type == TokenType::kMinus) { - if (peek_token_.type == TokenType::kInt) { + if (options_.fold_unary_operators && ops.size() > 1 && + ops[ops.size() - 2].type == TokenType::kMinus) { + // Match the ANTLR parser behavior where `-(-)+` prefers to match as + // repeated negate operators instead of a negation of an int literal. + // ---9223372036854775808 will fail to parse. + ops.pop_back(); + ops.pop_back(); + operand = ParseSelectorChain(); + } else if (peek_token_.type == TokenType::kInt) { int64_t op_id = ops.back().id; ops.pop_back(); operand = ParseNegativeIntLiteral(op_id); @@ -561,6 +569,13 @@ ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { for (int i = static_cast(ops.size()) - 1; i >= 0; --i) { std::vector args; + if (options_.fold_unary_operators && i > 0) { + if (ops[i - 1].type == ops[i].type) { + i--; + continue; + } + } + args.push_back(std::move(operand)); absl::string_view op_name = (ops[i].type == TokenType::kExclamation) ? CelOperator::LOGICAL_NOT diff --git a/parser/options.h b/parser/options.h index eb9d2eeea..1ddce71ac 100644 --- a/parser/options.h +++ b/parser/options.h @@ -85,6 +85,22 @@ struct ParserOptions final { // early testing of the Pratt parser. // TODO(b/527638023): Remove this option once the ANTLR parser is removed. bool enable_pratt_parser = false; + + // Folds repeated unary operators (!, -). + // + // If the operator appears repeatedly, the parser will ignore every contiguous + // pair. + // + // This makes it possible to parse some semantically invalid expressions as + // valid ones, though they are not particularly harmful. + // + // Examples that parse to the same AST: + // + // `---1` : `-(1)` + // `!!!!!true` : !`true` + // `--0u` : `0u` + // `!!"hello"` : `"hello"`). + bool fold_unary_operators = true; }; } // namespace cel diff --git a/parser/parser_test.cc b/parser/parser_test.cc index e2681608d..d4a3294c3 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -820,11 +820,7 @@ std::vector test_cases = { "", "", "", "", // PRATT PARSER AST "-_(\n" - " -_(\n" - " -_(\n" - " a^#4:Expr.Ident#\n" - " )^#3:Expr.Call#\n" - " )^#2:Expr.Call#\n" + " a^#4:Expr.Ident#\n" ")^#1:Expr.Call#"}, {"1 + +", "", "ERROR: :1:5: Syntax error: mismatched input '+' expecting {'[', " From 98a7da06d3b9492e10e22d624fe55d749ce137c0 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 24 Aug 2026 16:23:45 -0700 Subject: [PATCH 188/350] Make cel::Env and cel::EnvRuntime compiler and runtime builder methods const PiperOrigin-RevId: 970150654 --- env/env.cc | 5 +++-- env/env.h | 4 ++-- env/env_runtime.cc | 28 ++++++++++++++++++++-------- env/env_runtime.h | 8 ++++++-- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/env/env.cc b/env/env.cc index 85c5139da..dd71323d5 100644 --- a/env/env.cc +++ b/env/env.cc @@ -149,7 +149,8 @@ Env::Env() { compiler_options_.adapt_parser_errors = true; } -absl::StatusOr> Env::NewCompilerBuilder() { +absl::StatusOr> Env::NewCompilerBuilder() + const { CEL_ASSIGN_OR_RETURN( std::unique_ptr compiler_builder, cel::NewCompilerBuilder(descriptor_pool_, compiler_options_)); @@ -214,7 +215,7 @@ absl::StatusOr> Env::NewCompilerBuilder() { return compiler_builder; } -absl::StatusOr> Env::NewCompiler() { +absl::StatusOr> Env::NewCompiler() const { CEL_ASSIGN_OR_RETURN(std::unique_ptr compiler_builder, NewCompilerBuilder()); return compiler_builder->Build(); diff --git a/env/env.h b/env/env.h index 673ab19e4..c4a23e2cb 100644 --- a/env/env.h +++ b/env/env.h @@ -61,10 +61,10 @@ class Env { CompilerOptions& GetCompilerOptions() { return compiler_options_; } - absl::StatusOr> NewCompilerBuilder(); + absl::StatusOr> NewCompilerBuilder() const; // Shortcut for NewCompilerBuilder() followed by Build(). - absl::StatusOr> NewCompiler(); + absl::StatusOr> NewCompiler() const; private: cel::env_internal::ExtensionRegistry extension_registry_; diff --git a/env/env_runtime.cc b/env/env_runtime.cc index 33e0747cc..0f58d80b9 100644 --- a/env/env_runtime.cc +++ b/env/env_runtime.cc @@ -41,32 +41,38 @@ void EnvRuntime::RegisterExtensionFunctions( name, alias, version, std::move(function_registration_callback)); } -absl::StatusOr EnvRuntime::CreateRuntimeBuilder() { +absl::StatusOr EnvRuntime::CreateRuntimeBuilder() const { + return CreateRuntimeBuilder(runtime_options_); +} + +absl::StatusOr EnvRuntime::CreateRuntimeBuilder( + const RuntimeOptions& options) const { + RuntimeOptions runtime_options = options; const std::vector& extension_configs = config_.GetExtensionConfigs(); const Config::ExtensionConfig* optional_extension_config = nullptr; for (const Config::ExtensionConfig& extension_config : extension_configs) { if (extension_config.name == "optional") { optional_extension_config = &extension_config; - runtime_options_.enable_qualified_type_identifiers = true; + runtime_options.enable_qualified_type_identifiers = true; break; } } CEL_ASSIGN_OR_RETURN( RuntimeBuilder runtime_builder, - cel::CreateRuntimeBuilder(descriptor_pool_, runtime_options_)); + cel::CreateRuntimeBuilder(descriptor_pool_, runtime_options)); if (!config_.GetStandardLibraryConfig().disable) { CEL_RETURN_IF_ERROR(RegisterStandardFunctions( - runtime_builder.function_registry(), runtime_options_)); + runtime_builder.function_registry(), runtime_options)); } // Register optional extension functions first, because other extensions // depend on it (e.g. regex). if (optional_extension_config != nullptr) { CEL_RETURN_IF_ERROR(extension_registry_.RegisterExtensionFunctions( - runtime_builder, runtime_options_, optional_extension_config->name, + runtime_builder, runtime_options, optional_extension_config->name, optional_extension_config->version)); } @@ -75,14 +81,20 @@ absl::StatusOr EnvRuntime::CreateRuntimeBuilder() { continue; } CEL_RETURN_IF_ERROR(extension_registry_.RegisterExtensionFunctions( - runtime_builder, runtime_options_, extension_config.name, + runtime_builder, runtime_options, extension_config.name, extension_config.version)); } return runtime_builder; } -absl::StatusOr> EnvRuntime::NewRuntime() { - CEL_ASSIGN_OR_RETURN(RuntimeBuilder runtime_builder, CreateRuntimeBuilder()); +absl::StatusOr> EnvRuntime::NewRuntime() const { + return NewRuntime(runtime_options_); +} + +absl::StatusOr> EnvRuntime::NewRuntime( + const RuntimeOptions& options) const { + CEL_ASSIGN_OR_RETURN(RuntimeBuilder runtime_builder, + CreateRuntimeBuilder(options)); return std::move(runtime_builder).Build(); } diff --git a/env/env_runtime.h b/env/env_runtime.h index 63473c295..f8a80eb12 100644 --- a/env/env_runtime.h +++ b/env/env_runtime.h @@ -62,10 +62,14 @@ class EnvRuntime { RuntimeOptions& mutable_runtime_options() { return runtime_options_; } - absl::StatusOr CreateRuntimeBuilder(); + absl::StatusOr CreateRuntimeBuilder() const; + absl::StatusOr CreateRuntimeBuilder( + const RuntimeOptions& options) const; // Shortcut for CreateRuntimeBuilder() followed by Build(). - absl::StatusOr> NewRuntime(); + absl::StatusOr> NewRuntime() const; + absl::StatusOr> NewRuntime( + const RuntimeOptions& options) const; private: cel::env_internal::RuntimeExtensionRegistry& GetRuntimeExtensionRegistry() { From 9d1202c73e454c9f5d076da809bfb91ad579c0ed Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 25 Aug 2026 10:56:35 -0700 Subject: [PATCH 189/350] Change default behavior to drop function names from reference map. PiperOrigin-RevId: 970661818 --- checker/checker_options.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checker/checker_options.h b/checker/checker_options.h index cb85337fa..da6004d74 100644 --- a/checker/checker_options.h +++ b/checker/checker_options.h @@ -94,7 +94,7 @@ struct CheckerOptions { // // Temporary flag to allow rolling out the change. No functional changes to // evaluation behavior in either mode. - bool enable_function_name_in_reference = true; + bool enable_function_name_in_reference = false; // If true, the checker will use the proto json field names for protobuf // messages. Unlike protojson parsers, it will not accept the standard proto From 84b7f48d0f271badbf29b01da17e2748e67316e8 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 25 Aug 2026 11:15:44 -0700 Subject: [PATCH 190/350] Inline legacy message field access logic into select/optimized_select implementations. Refactor before adding switch over flag. PiperOrigin-RevId: 970674319 --- eval/eval/BUILD | 4 ++ eval/eval/select_step.cc | 40 +++++++++++++++--- extensions/BUILD | 5 ++- extensions/select_optimization.cc | 68 +++++++++++++++++++++++++++++-- 4 files changed, 106 insertions(+), 11 deletions(-) diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 329ee71f4..f6ce6e221 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -314,11 +314,15 @@ cc_library( ":direct_expression_step", ":evaluator_core", ":expression_step_base", + "//common:memory", "//common:type", "//common:value", "//common:value_kind", + "//eval/public:cel_value", + "//eval/public/structs:proto_message_type_adapter", "//internal:status_macros", "//runtime:runtime_options", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 0b31c3c13..a57179017 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -2,9 +2,11 @@ #include #include +#include #include #include +#include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" @@ -12,6 +14,7 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "common/legacy_value.h" +#include "common/memory.h" #include "common/type.h" #include "common/value.h" #include "common/value_kind.h" @@ -19,6 +22,8 @@ #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/public/cel_value.h" +#include "eval/public/structs/proto_message_type_adapter.h" #include "internal/status_macros.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -74,6 +79,29 @@ absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, return std::nullopt; } +// Helper for StructValue::GetFieldByName. Used for opting out of old reflection +// implementation. +absl::Status WrappedStructGet( + const Value& target, absl::string_view field, + ProtoWrapperTypeOptions unboxing_option, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) { + if (const google::protobuf::Message* message = + cel::interop_internal::GetLegacyMessage(target); + message != nullptr) { + CelValue::MessageWrapper message_wrapper( + message, &GetGenericProtoTypeInfoInstance()); + CEL_ASSIGN_OR_RETURN(CelValue cel_value, + internal::GetGenericProtoAccessApisInstance().GetField( + field, message_wrapper, unboxing_option, + cel::MemoryManagerRef::Pooling(arena))); + return cel::ModernValue(arena, cel_value, *result); + } + return target.GetStruct().GetFieldByName( + field, unboxing_option, descriptor_pool, message_factory, arena, result); +} + absl::Status PerformHas(const Value& target, absl::string_view field, const StringValue& field_value, const google::protobuf::DescriptorPool* descriptor_pool, @@ -115,9 +143,9 @@ absl::Status PerformGet(const Value& target, absl::string_view field, return absl::OkStatus(); } case ValueKind::kStruct: { - auto status = target.GetStruct().GetFieldByName( - field, unboxing_option, descriptor_pool, message_factory, arena, - &result); + auto status = + WrappedStructGet(target, field, unboxing_option, descriptor_pool, + message_factory, arena, &result); if (!status.ok()) { result = ErrorValue(std::move(status)); } @@ -154,9 +182,9 @@ absl::Status PerformOptionalGet(const Value& target, absl::string_view field, result = OptionalValue::None(); return absl::OkStatus(); } - CEL_RETURN_IF_ERROR(target.GetStruct().GetFieldByName( - field, unboxing_option, descriptor_pool, message_factory, arena, - &result)); + CEL_RETURN_IF_ERROR(WrappedStructGet(target, field, unboxing_option, + descriptor_pool, message_factory, + arena, &result)); ABSL_DCHECK(!result.IsUnknown()); result = OptionalValue::Of(std::move(result), arena); diff --git a/extensions/BUILD b/extensions/BUILD index faf9f08c8..df5477112 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -331,6 +331,7 @@ cc_library( "//common:expr", "//common:function_descriptor", "//common:kind", + "//common:memory", "//common:native_type", "//common:type", "//common:value", @@ -340,10 +341,13 @@ cc_library( "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/eval:expression_step_base", + "//eval/public:cel_value", + "//eval/public/structs:proto_message_type_adapter", "//internal:casts", "//internal:number", "//internal:status_macros", "//runtime:runtime_builder", + "//runtime:runtime_options", "//runtime/internal:errors", "//runtime/internal:runtime_friend_access", "//runtime/internal:runtime_impl", @@ -355,7 +359,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_absl//absl/types:variant", "@com_google_protobuf//:protobuf", diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 0cc64311a..83ea6abc6 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -18,8 +18,10 @@ #include #include #include +#include #include #include +#include #include #include "absl/algorithm/container.h" @@ -31,7 +33,6 @@ #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "absl/types/variant.h" #include "base/attribute.h" @@ -43,6 +44,8 @@ #include "common/expr.h" #include "common/function_descriptor.h" #include "common/kind.h" +#include "common/legacy_value.h" +#include "common/memory.h" #include "common/native_type.h" #include "common/type.h" #include "common/value.h" @@ -52,6 +55,8 @@ #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/public/cel_value.h" +#include "eval/public/structs/proto_message_type_adapter.h" #include "internal/casts.h" #include "internal/number.h" #include "internal/status_macros.h" @@ -59,6 +64,7 @@ #include "runtime/internal/runtime_friend_access.h" #include "runtime/internal/runtime_impl.h" #include "runtime/runtime_builder.h" +#include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -74,12 +80,15 @@ using ::cel::Expr; using ::cel::ExprKind; using ::cel::SelectExpr; using ::google::api::expr::runtime::AttributeTrail; +using ::google::api::expr::runtime::CelValue; using ::google::api::expr::runtime::DirectExpressionStep; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionFrameBase; using ::google::api::expr::runtime::ExpressionStepBase; +using ::google::api::expr::runtime::GetGenericProtoTypeInfoInstance; using ::google::api::expr::runtime::PlannerContext; using ::google::api::expr::runtime::ProgramOptimizer; +using ::google::api::expr::runtime::internal::GetGenericProtoAccessApisInstance; // Represents a single select operation (field access or indexing). // For struct-typed field accesses, includes the field name and the field @@ -267,6 +276,57 @@ absl::StatusOr MapKeyFromQualifier(const AttributeQualifier& qual, } } +// Helper for StructValue::GetFieldByName. Used for opting out of old reflection +// implementation. +absl::StatusOr WrappedStructGet( + const Value& target, absl::string_view field, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { + if (const google::protobuf::Message* message = + cel::interop_internal::GetLegacyMessage(target); + message != nullptr) { + CelValue::MessageWrapper message_wrapper( + message, &GetGenericProtoTypeInfoInstance()); + CEL_ASSIGN_OR_RETURN( + CelValue cel_value, + GetGenericProtoAccessApisInstance().GetField( + field, message_wrapper, ProtoWrapperTypeOptions::kUnsetProtoDefault, + MemoryManagerRef::Pooling(arena))); + Value result; + CEL_RETURN_IF_ERROR(cel::ModernValue(arena, cel_value, result)); + return result; + } + return target.GetStruct().GetFieldByName(field, descriptor_pool, + message_factory, arena); +} + +// Helper for StructValue::Qualify. Used for opting out of old reflection +// implementation. +absl::StatusOr> WrappedStructQualify( + const StructValue& struct_value, + absl::Span qualifiers, bool presence_test, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { + if (const google::protobuf::Message* message = + cel::interop_internal::GetLegacyMessage(struct_value); + message != nullptr) { + CelValue::MessageWrapper message_wrapper( + message, &GetGenericProtoTypeInfoInstance()); + CEL_ASSIGN_OR_RETURN(auto legacy_result, + GetGenericProtoAccessApisInstance().Qualify( + qualifiers, message_wrapper, presence_test, + MemoryManagerRef::Pooling(arena))); + Value result; + CEL_RETURN_IF_ERROR(cel::ModernValue(arena, legacy_result.value, result)); + return std::pair{std::move(result), + legacy_result.qualifier_count}; + } + return struct_value.Qualify(qualifiers, presence_test, descriptor_pool, + message_factory, arena); +} + absl::StatusOr ApplyQualifier( const Value& operand, const SelectQualifier& qualifier, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, @@ -280,8 +340,8 @@ absl::StatusOr ApplyQualifier( cel::runtime_internal::CreateNoMatchingOverloadError( "")); } return WrappedStructGet(operand, field_specifier.name, - descriptor_pool, message_factory, arena); + descriptor_pool, message_factory, arena, + enable_use_new_field_select_implementation); }, [&](const AttributeQualifier& qualifier) -> absl::StatusOr { if (operand.Is()) { @@ -371,15 +379,17 @@ absl::StatusOr FallbackSelect( bool presence_test, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) { + google::protobuf::Arena* absl_nonnull arena, + bool enable_use_new_field_select_implementation) { const Value* elem = &root; Value result; for (const auto& instruction : select_path.subspan(0, select_path.size() - 1)) { - CEL_ASSIGN_OR_RETURN(result, - ApplyQualifier(*elem, instruction, descriptor_pool, - message_factory, arena)); + CEL_ASSIGN_OR_RETURN( + result, + ApplyQualifier(*elem, instruction, descriptor_pool, message_factory, + arena, enable_use_new_field_select_implementation)); if (result->Is()) { return result; } @@ -417,7 +427,8 @@ absl::StatusOr FallbackSelect( } return ApplyQualifier(*elem, last_instruction, descriptor_pool, - message_factory, arena); + message_factory, arena, + enable_use_new_field_select_implementation); } absl::StatusOr> SelectInstructionsFromCall( @@ -692,15 +703,17 @@ absl::StatusOr OptimizedSelectImpl::ApplySelect( auto value_or = (options_.force_fallback_implementation) ? absl::UnimplementedError("Forced fallback impl") - : WrappedStructQualify(struct_value, select_path_, presence_test_, - frame.descriptor_pool(), - frame.message_factory(), frame.arena()); + : WrappedStructQualify( + struct_value, select_path_, presence_test_, + frame.descriptor_pool(), frame.message_factory(), frame.arena(), + frame.options().enable_use_new_field_select_implementation); if (!value_or.ok()) { if (value_or.status().code() == absl::StatusCode::kUnimplemented) { - return FallbackSelect(struct_value, select_path_, presence_test_, - frame.descriptor_pool(), frame.message_factory(), - frame.arena()); + return FallbackSelect( + struct_value, select_path_, presence_test_, frame.descriptor_pool(), + frame.message_factory(), frame.arena(), + frame.options().enable_use_new_field_select_implementation); } return value_or.status(); @@ -714,7 +727,8 @@ absl::StatusOr OptimizedSelectImpl::ApplySelect( value_or->first, absl::MakeConstSpan(select_path_).subspan(value_or->second), presence_test_, frame.descriptor_pool(), frame.message_factory(), - frame.arena()); + frame.arena(), + frame.options().enable_use_new_field_select_implementation); } AttributeTrail OptimizedSelectImpl::GetAttributeTrail( diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 072daf26c..245b8d52c 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -201,6 +201,16 @@ struct RuntimeOptions { // not what the planner expected (e.g. a map that was declared as a proto or // a different message with matching field names). bool enable_typed_field_access = false; + + // Temporary flag to gate using a new field selection implementation for + // protos. + // + // For the cel::Runtime APIs, this is a no-op. + // + // For google::api::expr::runtime::CelExpression, this will enable updated + // implementations for field access on protobuf messages, aligned with the + // cel::Value implementation. + bool enable_use_new_field_select_implementation = false; }; // LINT.ThenChange(//depot/google3/eval/public/cel_options.h) From f719d374cecdff1e68dee49ac95ea938fafc4c2e Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 27 Aug 2026 13:40:31 -0700 Subject: [PATCH 193/350] Default opt-in to updated reflection implementation. PiperOrigin-RevId: 972132029 --- eval/public/cel_options.h | 2 +- runtime/runtime_options.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index b0d9e6db0..a6e2692d4 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -232,7 +232,7 @@ struct InterpreterOptions { // For google::api::expr::runtime::CelExpression, this will enable updated // implementations for field access on protobuf messages, aligned with the // cel::Value implementation. - bool enable_use_new_field_select_implementation = false; + bool enable_use_new_field_select_implementation = true; }; // LINT.ThenChange(//depot/google3/runtime/runtime_options.h) diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 245b8d52c..831183dc8 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -210,7 +210,7 @@ struct RuntimeOptions { // For google::api::expr::runtime::CelExpression, this will enable updated // implementations for field access on protobuf messages, aligned with the // cel::Value implementation. - bool enable_use_new_field_select_implementation = false; + bool enable_use_new_field_select_implementation = true; }; // LINT.ThenChange(//depot/google3/eval/public/cel_options.h) From 11dba5a445fe3a0020650c1dd3c62630fb210939 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Fri, 28 Aug 2026 08:01:05 -0700 Subject: [PATCH 194/350] Default opt-in to updated reflection implementation. PiperOrigin-RevId: 972592895 --- eval/public/cel_options.h | 2 +- runtime/runtime_options.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index a6e2692d4..b0d9e6db0 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -232,7 +232,7 @@ struct InterpreterOptions { // For google::api::expr::runtime::CelExpression, this will enable updated // implementations for field access on protobuf messages, aligned with the // cel::Value implementation. - bool enable_use_new_field_select_implementation = true; + bool enable_use_new_field_select_implementation = false; }; // LINT.ThenChange(//depot/google3/runtime/runtime_options.h) diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 831183dc8..245b8d52c 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -210,7 +210,7 @@ struct RuntimeOptions { // For google::api::expr::runtime::CelExpression, this will enable updated // implementations for field access on protobuf messages, aligned with the // cel::Value implementation. - bool enable_use_new_field_select_implementation = true; + bool enable_use_new_field_select_implementation = false; }; // LINT.ThenChange(//depot/google3/eval/public/cel_options.h) From eeb74169957261d50bd9eb420faea40cc6aea603 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 28 Aug 2026 12:05:24 -0700 Subject: [PATCH 195/350] Consistency fixes for legacy adapter types: - make direct CelMap::Get calls return error as the legacy implementations did - avoid copying messages already marked as unsafe when converting to legacy PiperOrigin-RevId: 972717325 --- common/legacy_value.cc | 5 ++ common/legacy_value.h | 4 ++ common/values/legacy_map_value.cc | 61 ++++++++++++++--------- common/values/legacy_struct_value_test.cc | 12 +++++ common/values/parsed_map_field_value.h | 5 ++ common/values/parsed_message_value.h | 6 +++ eval/internal/BUILD | 3 +- eval/internal/cel_value_equal_test.cc | 13 +++-- 8 files changed, 79 insertions(+), 30 deletions(-) diff --git a/common/legacy_value.cc b/common/legacy_value.cc index a08b8317a..34b0e000f 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -260,6 +260,11 @@ CelValue LegacyTrivialStructValue(google::protobuf::Arena* absl_nonnull arena, } if (auto parsed_message_value = value.AsParsedMessage(); parsed_message_value) { + if (interop_internal::IsUnsafeParsedMessageValue(*parsed_message_value)) { + return CelValue::CreateMessageWrapper( + AsMessageWrapper(cel::to_address(*parsed_message_value), + &GetGenericProtoTypeInfoInstance())); + } auto maybe_cloned = parsed_message_value->Clone(arena); return CelValue::CreateMessageWrapper(MessageWrapper( cel::to_address(maybe_cloned), &GetGenericProtoTypeInfoInstance())); diff --git a/common/legacy_value.h b/common/legacy_value.h index a89eb0412..e71eb85a7 100644 --- a/common/legacy_value.h +++ b/common/legacy_value.h @@ -65,6 +65,10 @@ class MessageFactory; namespace cel::interop_internal { +inline bool IsUnsafeParsedMessageValue(const cel::ParsedMessageValue& value) { + return value.is_unsafe(); +} + // Returns the underlying `google::protobuf::Message` of a `cel::Value` if it is a legacy // message with the default type info, or `nullptr` otherwise. const google::protobuf::Message* absl_nullable GetLegacyMessage(const Value& value); diff --git a/common/values/legacy_map_value.cc b/common/values/legacy_map_value.cc index e287df076..8d9ce2ad3 100644 --- a/common/values/legacy_map_value.cc +++ b/common/values/legacy_map_value.cc @@ -22,7 +22,6 @@ #include #include "absl/base/nullability.h" -#include "absl/base/optimization.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -218,21 +217,33 @@ class LegacyParsedMapFieldMapValue final if (arena == nullptr) { arena = arena_; } - if (auto status = - google::api::expr::runtime::CelValue::CheckMapKeyType(key); - !status.ok()) { - status.IgnoreError(); - return std::nullopt; - } Value modern_key; - if (ABSL_PREDICT_FALSE(!ModernValue(arena, key, modern_key).ok())) { + if (!ModernValue(arena, key, modern_key).ok()) { + // Legacy to modern should succeed for a valid CelValue. return std::nullopt; } Value modern_val; - auto status_or_found = - Find(modern_key, google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), arena, &modern_val); - if (!status_or_found.ok() || !*status_or_found) { + // Call custom map Find directly. MapValue normally handles wrapping + // non-ok result to error value types, so emulate that here. + // + // Use the descriptor pool and message factory from the value. This is not + // totally consistent with modern APIs, but this should behave the same as + // the legacy map did. + const google::protobuf::Message* msg = value_.message_; + ABSL_DCHECK(msg->GetDescriptor() != nullptr); + ABSL_DCHECK(msg->GetReflection() != nullptr); + + const google::protobuf::DescriptorPool* descriptor_pool = + msg->GetDescriptor()->file()->pool(); + google::protobuf::MessageFactory* message_factory = + msg->GetReflection()->GetMessageFactory(); + auto found = + Find(modern_key, descriptor_pool, message_factory, arena, &modern_val); + if (!found.ok()) { + return google::api::expr::runtime::CreateErrorValue(arena, + found.status()); + } + if (!(*found) && !modern_val.IsError()) { return std::nullopt; } return UnsafeLegacyValue(modern_val, /*stable=*/false, arena); @@ -401,21 +412,25 @@ class LegacyParsedJsonMapValue final if (arena == nullptr) { arena = arena_; } - if (auto status = - google::api::expr::runtime::CelValue::CheckMapKeyType(key); - !status.ok()) { - status.IgnoreError(); - return std::nullopt; - } Value modern_key; - if (ABSL_PREDICT_FALSE(!ModernValue(arena, key, modern_key).ok())) { + if (!ModernValue(arena, key, modern_key).ok()) { + // Legacy to modern should succeed for a valid CelValue. return std::nullopt; } Value modern_val; - auto status_or_found = value_.Find( - modern_key, google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), arena, &modern_val); - if (!status_or_found.ok() || !*status_or_found) { + // Call custom map Find directly. MapValue normally handles wrapping + // non-ok result to error value types, so emulate that here. + // + // We know that the descriptor pool and message factory aren't needed here, + // so fine to use generated. + auto found = + Find(modern_key, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena, &modern_val); + if (!found.ok()) { + return google::api::expr::runtime::CreateErrorValue(arena, + found.status()); + } + if (!(*found) && !modern_val.IsError()) { return std::nullopt; } return UnsafeLegacyValue(modern_val, /*stable=*/false, arena); diff --git a/common/values/legacy_struct_value_test.cc b/common/values/legacy_struct_value_test.cc index 1170f2824..2e1949c0b 100644 --- a/common/values/legacy_struct_value_test.cc +++ b/common/values/legacy_struct_value_test.cc @@ -190,6 +190,18 @@ TEST_F(LegacyStructValueTest, MapFieldKeyTypeValidation) { CelValue str_key = CelValue::CreateString(&str_key_val); auto invalid_has_res = cel_map->Has(str_key); EXPECT_THAT(invalid_has_res, StatusIs(absl::StatusCode::kInvalidArgument)); + + auto invalid_get_res = cel_map->Get(arena(), str_key); + ASSERT_TRUE(invalid_get_res.has_value()); + ASSERT_TRUE(invalid_get_res->IsError()); + EXPECT_THAT(*invalid_get_res->ErrorOrDie(), + StatusIs(absl::StatusCode::kInvalidArgument)); + + auto invalid_subscript_res = (*cel_map)[str_key]; + ASSERT_TRUE(invalid_subscript_res.has_value()); + ASSERT_TRUE(invalid_subscript_res->IsError()); + EXPECT_THAT(*invalid_subscript_res->ErrorOrDie(), + StatusIs(absl::StatusCode::kInvalidArgument)); } TEST_F(LegacyStructValueTest, JsonStructAccess) { diff --git a/common/values/parsed_map_field_value.h b/common/values/parsed_map_field_value.h index 21d686bfd..f31f2e070 100644 --- a/common/values/parsed_map_field_value.h +++ b/common/values/parsed_map_field_value.h @@ -47,6 +47,10 @@ class ValueIterator; class ListValue; class ParsedJsonMapValue; +namespace common_internal { +class LegacyParsedMapFieldMapValue; +} // namespace common_internal + // ParsedMapFieldValue is a MapValue over a map field of a parsed protocol // buffer message. class ParsedMapFieldValue final @@ -192,6 +196,7 @@ class ParsedMapFieldValue final friend class ParsedJsonMapValue; friend class common_internal::ValueMixin; friend class common_internal::MapValueMixin; + friend class common_internal::LegacyParsedMapFieldMapValue; friend ParsedMapFieldValue UnsafeParsedMapFieldValue( const google::protobuf::Message* absl_nonnull message, const google::protobuf::FieldDescriptor* absl_nonnull field); diff --git a/common/values/parsed_message_value.h b/common/values/parsed_message_value.h index 3cad912e7..1d02a4369 100644 --- a/common/values/parsed_message_value.h +++ b/common/values/parsed_message_value.h @@ -48,6 +48,10 @@ namespace cel { +namespace interop_internal { +bool IsUnsafeParsedMessageValue(const ParsedMessageValue& value); +} + class MessageValue; class StructValue; class Value; @@ -189,6 +193,8 @@ class ParsedMessageValue final friend class common_internal::StructValueMixin; friend ParsedMessageValue UnsafeParsedMessageValue( const google::protobuf::Message* absl_nonnull value); + friend bool interop_internal::IsUnsafeParsedMessageValue( + const ParsedMessageValue& value); explicit ParsedMessageValue( const google::protobuf::Message* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND) diff --git a/eval/internal/BUILD b/eval/internal/BUILD index d6f31493e..c168ce3fa 100644 --- a/eval/internal/BUILD +++ b/eval/internal/BUILD @@ -56,11 +56,10 @@ cc_test( "//eval/public/structs:trivial_legacy_type_info", "//eval/testutil:test_message_cc_proto", "//internal:testing", - "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", "@com_google_absl//absl/types:span", - "@com_google_absl//absl/types:variant", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/eval/internal/cel_value_equal_test.cc b/eval/internal/cel_value_equal_test.cc index 109a63795..13787e973 100644 --- a/eval/internal/cel_value_equal_test.cc +++ b/eval/internal/cel_value_equal_test.cc @@ -18,20 +18,21 @@ #include #include #include +#include #include #include #include +#include #include #include "google/protobuf/any.pb.h" #include "google/rpc/context/attribute_context.pb.h" #include "google/protobuf/descriptor.pb.h" -#include "absl/status/statusor.h" +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "absl/types/span.h" -#include "absl/types/variant.h" #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_list_impl.h" #include "eval/public/containers/container_backed_map_impl.h" @@ -131,6 +132,8 @@ const std::vector& ValueExamples1() { result->push_back(CelValue::CreateMap(&CelMapExample1())); result->push_back(CelValue::CreateCelTypeView("type")); + ABSL_CHECK_EQ(arena.SpaceUsed(), 0) << "Arena should not be used."; + return result.release(); }(); return *examples; @@ -185,7 +188,7 @@ std::string CelValueEqualTestName( } TEST_P(CelValueEqualImplTypesTest, Basic) { - absl::optional result = CelValueEqualImpl(lhs(), rhs()); + std::optional result = CelValueEqualImpl(lhs(), rhs()); if (lhs().IsNull() || rhs().IsNull()) { if (lhs().IsNull() && rhs().IsNull()) { @@ -267,7 +270,7 @@ const std::vector& NumericValuesNotEqualExample() { using NumericInequalityTest = testing::TestWithParam; TEST_P(NumericInequalityTest, NumericValues) { NumericInequalityTestCase test_case = GetParam(); - absl::optional result = CelValueEqualImpl(test_case.a, test_case.b); + std::optional result = CelValueEqualImpl(test_case.a, test_case.b); EXPECT_TRUE(result.has_value()); EXPECT_EQ(*result, false); } @@ -280,7 +283,7 @@ INSTANTIATE_TEST_SUITE_P( }); TEST(CelValueEqualImplTest, LossyNumericEquality) { - absl::optional result = CelValueEqualImpl( + std::optional result = CelValueEqualImpl( CelValue::CreateDouble( static_cast(std::numeric_limits::max()) - 1), CelValue::CreateInt64(std::numeric_limits::max())); From c0ebb0614f4f5c82c769cba6111c6b65abde92e9 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 28 Aug 2026 12:25:58 -0700 Subject: [PATCH 196/350] [C++] Disallow trailing commas in function call argument lists in Pratt parser According to the CEL grammar specification, trailing commas are only permitted in collection literals (lists, maps, structs), not in function call argument lists (`exprList`). Disallows trailing commas in `PrattParserWorker::ParseArguments` and adds test cases to verify syntax error reporting. PiperOrigin-RevId: 972728668 --- parser/internal/pratt_parser_test.cc | 6 ++++++ parser/internal/pratt_parser_worker.h | 1 + parser/parser_test.cc | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 116ae607b..6c85dd1d7 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -1283,6 +1283,12 @@ std::vector GetErrorTestCases() { " | f(1, 2\n" " | ......^", }, + ErrorTestCase{ + .source = "foo(a,b,)", + .expected_error = "ERROR: :1:9: unexpected token\n" + " | foo(a,b,)\n" + " | ........^", + }, ErrorTestCase{ .source = "999999999999999999999999999999999999999", .expected_error = "ERROR: :1:1: invalid int literal\n" diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index dc17f27d6..f4886aa90 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -852,6 +852,7 @@ std::vector PrattParserWorker::ParseArguments( if (peek_token_.type == TokenType::kComma) { NextToken(); if (peek_token_.type == close_token) { + ReportError(peek_token_, "unexpected token"); break; } continue; diff --git a/parser/parser_test.cc b/parser/parser_test.cc index d4a3294c3..e47557e6b 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -501,6 +501,17 @@ std::vector test_cases = { "ERROR: :1:3: expected struct field name\n" " | t{>C}\n" " | ..^"}, + {"foo(a,b,)", "", + "ERROR: :1:9: Syntax error: mismatched input ')' expecting " + "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " + "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + " | foo(a,b,)\n" + " | ........^", + "", "", "", "", + // PRATT PARSER ERROR MESSAGE + "ERROR: :1:9: unexpected token\n" + " | foo(a,b,)\n" + " | ........^"}, // Macro tests {"has(m.f)", "m^#2:Expr.Ident#.f~test-only~^#4:Expr.Select#", "", From bf670878822d55e04537a16c619f5c8207d5a672 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 30 Aug 2026 23:58:03 -0700 Subject: [PATCH 197/350] No public description PiperOrigin-RevId: 973696886 --- eval/compiler/constant_folding.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/compiler/constant_folding.cc b/eval/compiler/constant_folding.cc index 118fc94c5..fa12a8e6d 100644 --- a/eval/compiler/constant_folding.cc +++ b/eval/compiler/constant_folding.cc @@ -96,7 +96,7 @@ class ConstantFoldingExtension : public ProgramOptimizer { static constexpr size_t kComprehensionSlotCount = 0; absl_nullable std::shared_ptr shared_arena_; - ABSL_ATTRIBUTE_UNUSED + [[maybe_unused]] absl_nullable std::shared_ptr shared_message_factory_; Activation empty_; FlatExpressionEvaluatorState state_; From fddab74ea4d1eff8b3fd2cc1251f4c11d416db48 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 31 Aug 2026 08:04:40 -0700 Subject: [PATCH 198/350] No public description PiperOrigin-RevId: 973885731 --- runtime/internal/runtime_env.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/internal/runtime_env.h b/runtime/internal/runtime_env.h index cb9d9b93d..efc68cd5f 100644 --- a/runtime/internal/runtime_env.h +++ b/runtime/internal/runtime_env.h @@ -70,7 +70,7 @@ struct RuntimeEnv final { bool IsInitialized() const { return well_known_types.IsInitialized(); } - ABSL_ATTRIBUTE_UNUSED + [[maybe_unused]] const absl_nonnull std::shared_ptr descriptor_pool; From f7bced5f0d9ba9c8089c13ccec88bae875023353 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 31 Aug 2026 08:12:36 -0700 Subject: [PATCH 199/350] No public description PiperOrigin-RevId: 973889644 --- eval/public/ast_rewrite.cc | 2 +- eval/public/ast_traverse.cc | 2 +- eval/public/cel_function_adapter_impl.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eval/public/ast_rewrite.cc b/eval/public/ast_rewrite.cc index 87c667eb5..86f5ee4d1 100644 --- a/eval/public/ast_rewrite.cc +++ b/eval/public/ast_rewrite.cc @@ -72,7 +72,7 @@ using StackRecordKind = struct StackRecord { public: - ABSL_ATTRIBUTE_UNUSED static constexpr int kNotCallArg = -1; + [[maybe_unused]] static constexpr int kNotCallArg = -1; static constexpr int kTarget = -2; StackRecord(Expr* e, const SourceInfo* info) { diff --git a/eval/public/ast_traverse.cc b/eval/public/ast_traverse.cc index a31e1d37d..eeab65498 100644 --- a/eval/public/ast_traverse.cc +++ b/eval/public/ast_traverse.cc @@ -72,7 +72,7 @@ using StackRecordKind = struct StackRecord { public: - ABSL_ATTRIBUTE_UNUSED static constexpr int kNotCallArg = -1; + [[maybe_unused]] static constexpr int kNotCallArg = -1; static constexpr int kTarget = -2; StackRecord(const Expr* e, const SourceInfo* info) { diff --git a/eval/public/cel_function_adapter_impl.h b/eval/public/cel_function_adapter_impl.h index 6cd661c10..d4ecd54bd 100644 --- a/eval/public/cel_function_adapter_impl.h +++ b/eval/public/cel_function_adapter_impl.h @@ -346,9 +346,9 @@ class FunctionAdapterImpl { #else inline absl::Status RunWrap( std::function func, - ABSL_ATTRIBUTE_UNUSED const absl::Span argset, + [[maybe_unused]] const absl::Span argset, ::google::protobuf::Arena* arena, CelValue* result, - ABSL_ATTRIBUTE_UNUSED int arg_index) const { + [[maybe_unused]] int arg_index) const { return ValueConverter().NativeToValue(func(), arena, result); } From aa98a1a0a0ed1a0ac19e70b9aebd1c8de75b825a Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 31 Aug 2026 08:13:40 -0700 Subject: [PATCH 200/350] No public description PiperOrigin-RevId: 973890078 --- common/type_kind.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/type_kind.h b/common/type_kind.h index 34df8e385..01dcce3bb 100644 --- a/common/type_kind.h +++ b/common/type_kind.h @@ -77,7 +77,7 @@ constexpr Kind TypeKindToKind(TypeKind kind) { return static_cast(static_cast>(kind)); } -constexpr bool KindIsTypeKind(Kind kind ABSL_ATTRIBUTE_UNUSED) { +constexpr bool KindIsTypeKind(Kind kind [[maybe_unused]]) { // Currently all Kind are valid TypeKind. return true; } From 1953464b870b6acf804d3abb421b8710cbac650e Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 31 Aug 2026 08:18:48 -0700 Subject: [PATCH 201/350] No public description PiperOrigin-RevId: 973892331 --- eval/public/structs/legacy_type_provider.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/public/structs/legacy_type_provider.h b/eval/public/structs/legacy_type_provider.h index b00645d0e..b04f7af36 100644 --- a/eval/public/structs/legacy_type_provider.h +++ b/eval/public/structs/legacy_type_provider.h @@ -53,7 +53,7 @@ class LegacyTypeProvider { // created ones, the TypeInfoApis returned from this method should be the same // as the ones used in value creation. virtual std::optional ProvideLegacyTypeInfo( - ABSL_ATTRIBUTE_UNUSED absl::string_view name) const { + [[maybe_unused]] absl::string_view name) const { return std::nullopt; } }; From 35195b21cafe814aca08ad3ab96e795c250f4a4b Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 31 Aug 2026 08:23:10 -0700 Subject: [PATCH 202/350] No public description PiperOrigin-RevId: 973894126 --- internal/overflow.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/overflow.cc b/internal/overflow.cc index 8cc209384..79d1f24ad 100644 --- a/internal/overflow.cc +++ b/internal/overflow.cc @@ -32,7 +32,7 @@ constexpr int64_t kInt32Min = std::numeric_limits::lowest(); constexpr int64_t kInt64Max = std::numeric_limits::max(); constexpr int64_t kInt64Min = std::numeric_limits::lowest(); constexpr uint64_t kUint32Max = std::numeric_limits::max(); -ABSL_ATTRIBUTE_UNUSED constexpr uint64_t kUint64Max = +[[maybe_unused]] constexpr uint64_t kUint64Max = std::numeric_limits::max(); constexpr uint64_t kUintToIntMax = static_cast(kInt64Max); constexpr double kDoubleToIntMax = static_cast(kInt64Max); From 27f54d4487831c157bef151d0ccd23d7431bfd7f Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 31 Aug 2026 12:41:59 -0700 Subject: [PATCH 203/350] [Pratt parser] Fix operation precedence following negative numbers: `-42.foo` PiperOrigin-RevId: 974023485 --- parser/internal/pratt_parser_test.cc | 66 +++++++++++++++++++++++++++ parser/internal/pratt_parser_worker.h | 2 + parser/parser_test.cc | 21 +++++++++ 3 files changed, 89 insertions(+) diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 6c85dd1d7..908690bc8 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -482,6 +482,72 @@ std::vector GetParserTestCases() { )^#1:Expr.Call# )", }, + TestCase{ + .source = "-.2.V", + .expected_ast = R"( + -0.2^#1:double#.V^#2:Expr.Select# + )", + }, + TestCase{ + .source = "!-.2.V", + .expected_ast = R"( + !_( + -0.2^#2:double#.V^#3:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-2.V", + .expected_ast = R"( + !_( + -2^#2:int64#.V^#3:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-.2[0]", + .expected_ast = R"( + !_( + _[_]( + -0.2^#2:double#, + 0^#4:int64# + )^#3:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-x.foo[0]", + .expected_ast = R"( + -_( + _[_]( + x^#2:Expr.Ident#.foo^#3:Expr.Select#, + 0^#5:int64# + )^#4:Expr.Call# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "-x[0].foo", + .expected_ast = R"( + -_( + _[_]( + x^#2:Expr.Ident#, + 0^#4:int64# + )^#3:Expr.Call#.foo^#5:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!x[0].foo", + .expected_ast = R"( + !_( + _[_]( + x^#2:Expr.Ident#, + 0^#4:int64# + )^#3:Expr.Call#.foo^#5:Expr.Select# + )^#1:Expr.Call# + )", + }, TestCase{ .source = "a + b", .expected_ast = R"( diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index f4886aa90..1f99984c5 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -556,10 +556,12 @@ ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { int64_t op_id = ops.back().id; ops.pop_back(); operand = ParseNegativeIntLiteral(op_id); + ParseSelectorChainTail(operand); } else if (peek_token_.type == TokenType::kFloat) { int64_t op_id = ops.back().id; ops.pop_back(); operand = ParseNegativeDoubleLiteral(op_id); + ParseSelectorChainTail(operand); } else { operand = ParseSelectorChain(); } diff --git a/parser/parser_test.cc b/parser/parser_test.cc index e47557e6b..dbac2b02f 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -137,6 +137,27 @@ std::vector test_cases = { " a^#1:Expr.Ident#,\n" " 3^#3:int64#\n" ")^#2:Expr.Call#"}, + {"-x.foo[0]", + "-_(\n" + " _[_](\n" + " x^#2:Expr.Ident#.foo^#3:Expr.Select#,\n" + " 0^#5:int64#\n" + " )^#4:Expr.Call#\n" + ")^#1:Expr.Call#"}, + {"-x[0].foo", + "-_(\n" + " _[_](\n" + " x^#2:Expr.Ident#,\n" + " 0^#4:int64#\n" + " )^#3:Expr.Call#.foo^#5:Expr.Select#\n" + ")^#1:Expr.Call#"}, + {"!x[0].foo", + "!_(\n" + " _[_](\n" + " x^#2:Expr.Ident#,\n" + " 0^#4:int64#\n" + " )^#3:Expr.Call#.foo^#5:Expr.Select#\n" + ")^#1:Expr.Call#"}, {"SomeMessage{foo: 5, bar: \"xyz\"}", "SomeMessage{\n" " foo:5^#3:int64#^#2:Expr.CreateStruct.Entry#,\n" From 71194a388a2318f4768e2a685b8a019209811e7a Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 31 Aug 2026 14:27:13 -0700 Subject: [PATCH 204/350] Support match_path for non-repeated primitive fields, maps, and message. PiperOrigin-RevId: 974079397 --- tools/proto_to_predicate.cc | 19 +++++-- tools/proto_to_predicate_test.cc | 97 ++++++++++++++++++++++++++++++++ tools/testdata/BUILD | 1 + tools/testdata/test_policy.proto | 27 +++++++++ 4 files changed, 140 insertions(+), 4 deletions(-) diff --git a/tools/proto_to_predicate.cc b/tools/proto_to_predicate.cc index 8c89ee2f0..01f35fd05 100644 --- a/tools/proto_to_predicate.cc +++ b/tools/proto_to_predicate.cc @@ -103,6 +103,17 @@ class ProtoToPredicateBuilder final : private ExprFactory { } return e; } + + // Returns either the path specified by the "match_path" annotation, + // or the default path derived from the field name. + Expr GetFieldPath(const Expr& base_expr, + const ::google::protobuf::FieldDescriptor* field) { + std::string match_path_val = GetMatchPath(field); + if (!match_path_val.empty()) { + return ParseAndBuildPath(match_path_val); + } + return NewSelect(NextId(), base_expr, field->name()); + } ExprId NextId() { return id_++; } // --------------------------------------------------------------------------- @@ -246,7 +257,7 @@ class ProtoToPredicateBuilder final : private ExprFactory { const FieldDescriptor* const value_field = field->message_type()->FindFieldByName("value"); - Expr map_path = NewSelect(NextId(), base_expr, field->name()); + Expr map_path = GetFieldPath(base_expr, field); struct MapEntry { const Message* message; @@ -355,7 +366,7 @@ class ProtoToPredicateBuilder final : private ExprFactory { const Message& sub_message = reflection->GetRepeatedMessage(message, field, i); std::vector sub_predicates; - Expr sub_base = NewSelect(NextId(), base_expr, field->name()); + Expr sub_base = GetFieldPath(base_expr, field); CEL_RETURN_IF_ERROR(Walk(sub_message, sub_base, sub_predicates)); message_asts.push_back(LogicalAnd(sub_predicates)); } @@ -426,11 +437,11 @@ class ProtoToPredicateBuilder final : private ExprFactory { } } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { const Message& sub_message = reflection->GetMessage(message, field); - Expr field_path = NewSelect(NextId(), base_expr, field->name()); + Expr field_path = GetFieldPath(base_expr, field); CEL_RETURN_IF_ERROR(Walk(sub_message, field_path, predicates)); } else { // Primitive field: base_expr.field == - Expr field_path = NewSelect(NextId(), base_expr, field->name()); + Expr field_path = GetFieldPath(base_expr, field); predicates.push_back( ConstructEquality(std::move(field_path), PrimitiveToExpr(message, reflection, field))); diff --git a/tools/proto_to_predicate_test.cc b/tools/proto_to_predicate_test.cc index 80ad140c7..2e7b94f6f 100644 --- a/tools/proto_to_predicate_test.cc +++ b/tools/proto_to_predicate_test.cc @@ -549,6 +549,103 @@ INSTANTIATE_TEST_SUITE_P( .json_input = R"({ "destinations": [ { "tool": { } } ] })", .expected_unparsed = "true", }, + PolicyTestCase{ + .name = "AnnotatedSingularFieldInMessage", + .json_input = + R"({ "destinations": [ { + "agent": { + "id": "agent-007", + "location": "us-central1" + } + } ] })", + .expected_unparsed = "dest.agent.name == \"agent-007\" && " + "custom.agent.location == \"us-central1\"", + }, + PolicyTestCase{ + .name = "AnnotatedOneofPrimitiveField", + .json_input = + R"({ "destinations": [ { + "ip": "192.168.1.1" + } ] })", + .expected_unparsed = "custom.ip == \"192.168.1.1\"", + }, + PolicyTestCase{ + .name = "AnnotatedMapFieldPrimitive", + .json_input = + R"({ "destinations": [ { + "tool": { + "annotated_labels": { + "cluster": "us-central1" + } + } + } ] })", + .expected_unparsed = + "\"cluster\" in custom.labels && " + "custom.labels[\"cluster\"] == \"us-central1\"", + }, + PolicyTestCase{ + .name = "AnnotatedMapFieldMessage", + .json_input = + R"({ "destinations": [ { + "tool": { + "annotated_role_members": { + "admin": { + "all_users": true, + "principals": ["alice_user"], + "leader": "alice", + "leaders": ["bob"] + } + } + } + } ] })", + .expected_unparsed = + "\"admin\" in custom.role_members && " + "\"alice_user\" in custom.role_members[\"admin\"].principals " + "&& " + "custom.role_members[\"admin\"].all_users == true && " + "custom.member.leader == \"alice\" && " + "custom.member.leaders in [\"bob\"]", + }, + PolicyTestCase{ + .name = "AnnotatedMessageField", + .json_input = + R"({ "destinations": [ { + "tool": { + "annotated_annotations": { + "read_only_hint": true + } + } + } ] })", + .expected_unparsed = + "custom.tool_annotations.read_only_hint == true", + }, + PolicyTestCase{ + .name = "AnnotatedRepeatedMessageFieldSingle", + .json_input = + R"({ "destinations": [ { + "tool": { + "backup_agents": [ + { "id": "agent-007" } + ] + } + } ] })", + .expected_unparsed = "custom.backup_agents.name == \"agent-007\"", + }, + PolicyTestCase{ + .name = "AnnotatedRepeatedMessageFieldMultiple", + .json_input = + R"({ "destinations": [ { + "tool": { + "backup_agents": [ + { "id": "agent-007" }, + { "id": "agent-008" } + ] + } + } ] })", + .expected_unparsed = + "custom.backup_agents.name == \"agent-007\" || " + "custom.backup_agents.name == \"agent-008\"", + }, PolicyTestCase{ .name = "MapEquality", .json_input = diff --git a/tools/testdata/BUILD b/tools/testdata/BUILD index f9d8ea4b9..956a13eaf 100644 --- a/tools/testdata/BUILD +++ b/tools/testdata/BUILD @@ -32,6 +32,7 @@ proto_library( name = "test_policy_proto", srcs = ["test_policy.proto"], visibility = ["//tools:__subpackages__"], + deps = ["@com_google_protobuf//:descriptor_proto"], ) cc_proto_library( diff --git a/tools/testdata/test_policy.proto b/tools/testdata/test_policy.proto index b5d424c04..f0ce9fe76 100644 --- a/tools/testdata/test_policy.proto +++ b/tools/testdata/test_policy.proto @@ -19,11 +19,18 @@ edition = "2023"; package cel.cpp.tools; +import "google/protobuf/descriptor.proto"; + option cc_enable_arenas = true; +extend google.protobuf.FieldOptions { + string match_path = 51074; +} + // Represents the targeted client agent. message Agent { string name = 1 [json_name = "id"]; + string location = 2 [(match_path) = "custom.agent.location"]; } // Specifies additional metadata tool annotations. @@ -40,6 +47,10 @@ message Members { bool all_users = 3; bool all_authenticated_users = 4; + + string leader = 5 [(match_path) = "custom.member.leader"]; + + repeated string leaders = 6 [(match_path) = "custom.member.leaders"]; } // Represents a metadata tool block. @@ -56,6 +67,21 @@ message Tool { // A map with string keys representing roles and Member instances as values. map role_members = 4; + + // A map with string keys representing roles and Member instances as values, + // annotated with a match_path. + map annotated_role_members = 5 + [(match_path) = "custom.role_members"]; + + // A string-to-string map, annotated with a match_path. + map annotated_labels = 6 [(match_path) = "custom.labels"]; + + // A non-repeated message field, annotated with a match_path. + ToolAnnotations annotated_annotations = 7 + [(match_path) = "custom.tool_annotations"]; + + // A repeated message field, annotated with a match_path. + repeated Agent backup_agents = 8 [(match_path) = "custom.backup_agents"]; } // Represents a policy mapping destination block. @@ -63,6 +89,7 @@ message Target { oneof kind { Agent agent = 1; Tool tool = 2; + string ip = 3 [(match_path) = "custom.ip"]; } } From 973fc88c4fcb6569c390f3f403cf7e292c164078 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 31 Aug 2026 14:54:19 -0700 Subject: [PATCH 205/350] [Pratt parser] Improve compatibility with ANTLR parser PiperOrigin-RevId: 974093125 --- parser/internal/pratt_parser_test.cc | 176 ++++++++++++++----------- parser/internal/pratt_parser_worker.cc | 9 +- parser/internal/pratt_parser_worker.h | 39 +++--- parser/parser_test.cc | 30 ++--- 4 files changed, 143 insertions(+), 111 deletions(-) diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 908690bc8..422a5d19f 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -1222,21 +1222,24 @@ std::vector GetErrorTestCases() { ErrorTestCase{ .source = "1 + 2 * 3 4", .expected_error = - "ERROR: :1:11: unexpected token after expression\n" + "ERROR: :1:11: Syntax error: unexpected token after " + "expression\n" " | 1 + 2 * 3 4\n" " | ..........^", }, ErrorTestCase{ .source = "1{}", .expected_error = - "ERROR: :1:2: unexpected token after expression\n" + "ERROR: :1:2: Syntax error: unexpected token after " + "expression\n" " | 1{}\n" " | .^", }, ErrorTestCase{ .source = "true ? 1", .expected_error = - "ERROR: :1:9: expected ':' in conditional expression\n" + "ERROR: :1:9: Syntax error: expected ':' in conditional " + "expression\n" " | true ? 1\n" " | ........^", }, @@ -1248,10 +1251,10 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "a.", - .expected_error = - "ERROR: :1:3: expected identifier after '.'\n" - " | a.\n" - " | ..^", + .expected_error = "ERROR: :1:3: Syntax error: expected " + "identifier after '.'\n" + " | a.\n" + " | ..^", }, ErrorTestCase{ .source = "a[?0]", @@ -1261,9 +1264,10 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = ". *", - .expected_error = "ERROR: :1:3: expected identifier\n" - " | . *\n" - " | ..^", + .expected_error = + "ERROR: :1:3: Syntax error: expected identifier\n" + " | . *\n" + " | ..^", }, ErrorTestCase{ .source = ".as", @@ -1274,17 +1278,19 @@ std::vector GetErrorTestCases() { ErrorTestCase{ .source = "* 2", .expected_error = - "ERROR: :1:1: unexpected token\n" + "ERROR: :1:1: Syntax error: unexpected token\n" " | * 2\n" " | ^\n" - "ERROR: :1:3: unexpected token after expression\n" + "ERROR: :1:3: Syntax error: unexpected token after " + "expression\n" " | * 2\n" " | ..^", }, ErrorTestCase{ .source = "(1 + 2", .expected_error = - "ERROR: :1:7: mismatched input expecting ')'\n" + "ERROR: :1:7: Syntax error: mismatched input " + "expecting ')'\n" " | (1 + 2\n" " | ......^", }, @@ -1296,7 +1302,7 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "[1, 2", - .expected_error = "ERROR: :1:6: expected ']'\n" + .expected_error = "ERROR: :1:6: Syntax error: expected ']'\n" " | [1, 2\n" " | .....^", }, @@ -1308,13 +1314,14 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "{'k' 'v'}", - .expected_error = "ERROR: :1:6: expected ':' in map entry\n" - " | {'k' 'v'}\n" - " | .....^", + .expected_error = + "ERROR: :1:6: Syntax error: expected ':' in map entry\n" + " | {'k' 'v'}\n" + " | .....^", }, ErrorTestCase{ .source = "{'k': 'v'", - .expected_error = "ERROR: :1:10: expected '}'\n" + .expected_error = "ERROR: :1:10: Syntax error: expected '}'\n" " | {'k': 'v'\n" " | .........^", }, @@ -1326,26 +1333,29 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "Msg{1: 2}", - .expected_error = "ERROR: :1:5: expected struct field name\n" - " | Msg{1: 2}\n" - " | ....^", + .expected_error = + "ERROR: :1:5: Syntax error: expected struct field name\n" + " | Msg{1: 2}\n" + " | ....^", }, ErrorTestCase{ .source = "Msg{f 10}", - .expected_error = "ERROR: :1:7: expected ':' in struct field\n" - " | Msg{f 10}\n" - " | ......^", + .expected_error = + "ERROR: :1:7: Syntax error: expected ':' in struct field\n" + " | Msg{f 10}\n" + " | ......^", }, ErrorTestCase{ .source = "Msg{f: 10", - .expected_error = "ERROR: :1:10: expected '}'\n" + .expected_error = "ERROR: :1:10: Syntax error: expected '}'\n" " | Msg{f: 10\n" " | .........^", }, ErrorTestCase{ .source = "f(1, 2", .expected_error = - "ERROR: :1:7: mismatched input expecting ')'\n" + "ERROR: :1:7: Syntax error: mismatched input " + "expecting ')'\n" " | f(1, 2\n" " | ......^", }, @@ -1357,35 +1367,39 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "999999999999999999999999999999999999999", - .expected_error = "ERROR: :1:1: invalid int literal\n" - " | 999999999999999999999999999999999999999\n" - " | ^", + .expected_error = + "ERROR: :1:1: Syntax error: invalid int literal\n" + " | 999999999999999999999999999999999999999\n" + " | ^", }, ErrorTestCase{ .source = "999999999999999999999999999999999999999u", - .expected_error = "ERROR: :1:1: invalid uint literal\n" - " | 999999999999999999999999999999999999999u\n" - " | ^", + .expected_error = + "ERROR: :1:1: Syntax error: invalid uint literal\n" + " | 999999999999999999999999999999999999999u\n" + " | ^", }, ErrorTestCase{ .source = "1e", .expected_error = - "ERROR: :1:1: floating point literal missing digits after " - "exponent separator\n" + "ERROR: :1:1: Syntax error: floating point literal " + "missing digits after exponent separator\n" " | 1e\n" " | ^", }, ErrorTestCase{ .source = "\"unterminated", - .expected_error = "ERROR: :1:1: unterminated string literal\n" - " | \"unterminated\n" - " | ^", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"unterminated\n" + " | ^", }, ErrorTestCase{ .source = "b\"unterminated", - .expected_error = "ERROR: :1:1: unterminated bytes literal\n" - " | b\"unterminated\n" - " | ^", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated bytes literal\n" + " | b\"unterminated\n" + " | ^", }, ErrorTestCase{ .source = "a.?`foo`", @@ -1440,7 +1454,8 @@ std::vector GetErrorTestCases() { ErrorTestCase{ .source = "`foo", .expected_error = - "ERROR: :1:1: unterminated quoted identifier\n" + "ERROR: :1:1: Syntax error: unterminated quoted " + "identifier\n" " | `foo\n" " | ^", .enable_quoted_identifiers = true, @@ -1448,73 +1463,82 @@ std::vector GetErrorTestCases() { ErrorTestCase{ .source = "f(*, 1e, {2 3})", .expected_error = - "ERROR: :1:3: unexpected token\n" + "ERROR: :1:3: Syntax error: unexpected token\n" " | f(*, 1e, {2 3})\n" " | ..^\n" - "ERROR: :1:6: floating point literal missing digits after " - "exponent separator\n" + "ERROR: :1:6: Syntax error: floating point literal " + "missing digits after exponent separator\n" " | f(*, 1e, {2 3})\n" " | .....^\n" - "ERROR: :1:13: expected ':' in map entry\n" + "ERROR: :1:13: Syntax error: expected ':' in map entry\n" " | f(*, 1e, {2 3})\n" " | ............^", }, ErrorTestCase{ .source = "(1 + *) + 2", - .expected_error = "ERROR: :1:6: unexpected token\n" - " | (1 + *) + 2\n" - " | .....^", + .expected_error = + "ERROR: :1:6: Syntax error: unexpected token\n" + " | (1 + *) + 2\n" + " | .....^", }, ErrorTestCase{ .source = "f(1 + *, 2)", - .expected_error = "ERROR: :1:7: unexpected token\n" - " | f(1 + *, 2)\n" - " | ......^", + .expected_error = + "ERROR: :1:7: Syntax error: unexpected token\n" + " | f(1 + *, 2)\n" + " | ......^", }, ErrorTestCase{ .source = "(a. + 1)", .expected_error = - "ERROR: :1:5: expected identifier after '.'\n" + "ERROR: :1:5: Syntax error: expected identifier after " + "'.'\n" " | (a. + 1)\n" " | ....^", }, ErrorTestCase{ .source = "f(a., 1)", .expected_error = - "ERROR: :1:5: expected identifier after '.'\n" + "ERROR: :1:5: Syntax error: expected identifier after " + "'.'\n" " | f(a., 1)\n" " | ....^", }, ErrorTestCase{ .source = "[a., 1]", .expected_error = - "ERROR: :1:4: expected identifier after '.'\n" + "ERROR: :1:4: Syntax error: expected identifier after " + "'.'\n" " | [a., 1]\n" " | ...^", }, ErrorTestCase{ .source = "-0x8000000000000001", - .expected_error = "ERROR: :1:2: invalid int literal\n" - " | -0x8000000000000001\n" - " | .^", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -0x8000000000000001\n" + " | .^", }, ErrorTestCase{ .source = "-0x10000000000000000", - .expected_error = "ERROR: :1:2: invalid int literal\n" - " | -0x10000000000000000\n" - " | .^", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -0x10000000000000000\n" + " | .^", }, ErrorTestCase{ .source = "-9223372036854775809", - .expected_error = "ERROR: :1:2: invalid int literal\n" - " | -9223372036854775809\n" - " | .^", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -9223372036854775809\n" + " | .^", }, ErrorTestCase{ .source = "-999999999999999999999999999999999999999", - .expected_error = "ERROR: :1:2: invalid int literal\n" - " | -999999999999999999999999999999999999999\n" - " | .^", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -999999999999999999999999999999999999999\n" + " | .^", }, ErrorTestCase{ .source = "-", @@ -1526,15 +1550,17 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "- *", - .expected_error = "ERROR: :1:3: unexpected token\n" - " | - *\n" - " | ..^", + .expected_error = + "ERROR: :1:3: Syntax error: unexpected token\n" + " | - *\n" + " | ..^", }, ErrorTestCase{ .source = "\"😀😀😀😀😀\" ~error", - .expected_error = "ERROR: :1:9: unexpected character\n" - " | \"😀😀😀😀😀\" ~error\n" - " | ........^", + .expected_error = + "ERROR: :1:9: Syntax error: unexpected character\n" + " | \"😀😀😀😀😀\" ~error\n" + " | ........^", }, }; } @@ -1892,9 +1918,9 @@ TEST(PrattParserMacroErrorTest, ReportError) { auto ast = parser->Parse(*source, &issues); EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); EXPECT_EQ(FormatIssues(*source, issues), - "ERROR: :1:6: custom macro error\n" + "ERROR: :1:15: custom macro error\n" " | 42 + bad_macro(x)\n" - " | .....^"); + " | ..............^"); } TEST(PrattParserMacroErrorTest, ReportErrorAt) { @@ -1941,7 +1967,7 @@ TEST(PrattParserErrorRecoveryTest, ErrorRecoveryLimitOne) { ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("......")); EXPECT_EQ(FormatIssues(*source, issues), "ERROR: :-1:0: Error recovery limit (1) exceeded\n" - "ERROR: :1:2: expected identifier\n" + "ERROR: :1:2: Syntax error: expected identifier\n" " | ......\n" " | .^"); } diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index b7356528d..198a70c4f 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -133,7 +133,7 @@ Token ParserWorker::NextSignificantToken(bool report_error) { continue; } if (tok.type == TokenType::kError && report_error) { - ReportError(tok, lexer_.GetError().message); + ReportSyntaxError(tok, lexer_.GetError().message); if (is_recovery_limit_exceeded()) { return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; } @@ -177,7 +177,7 @@ bool ParserWorker::Expect(TokenType type, absl::string_view msg) { } else { err_msg = std::string(msg); } - ReportError(peek_token_, err_msg); + ReportSyntaxError(peek_token_, err_msg); } SynchronizeOnDelimiter(); return false; @@ -271,4 +271,9 @@ void ParserWorker::ReportError(const SourceLocation& loc, } } +void ParserWorker::ReportSyntaxError(const Token& token, + absl::string_view msg) { + ReportError(token.start, absl::StrCat("Syntax error: ", msg)); +} + } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 1f99984c5..966717af1 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -110,6 +110,7 @@ class ParserWorker { void ReportError(const Token& token, absl::string_view msg) { ReportError(token.start, msg); } + void ReportSyntaxError(const Token& token, absl::string_view msg); const cel::Source& source_; cel::ParserOptions options_; @@ -335,7 +336,7 @@ ExprNode PrattParserWorker::Parse() { } if (peek_token_.type != TokenType::kEnd && peek_token_.type != TokenType::kError) { - ReportError(peek_token_, "unexpected token after expression"); + ReportSyntaxError(peek_token_, "unexpected token after expression"); } return expr; } @@ -357,8 +358,8 @@ ExprNode PrattParserWorker::ParseExpr() { template void PrattParserWorker::ParseTernary(ExprNode& lhs) { - NextToken(); - int64_t op_id = NextId(); + Token op_tok = NextToken(); + int64_t op_id = NextId(op_tok); ExprNode true_expr = ParseBinaryAndTernary(1); if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { return; @@ -462,7 +463,7 @@ void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { if (id_tok.type != TokenType::kIdent && id_tok.type != TokenType::kReservedWord) { if (id_tok.type != TokenType::kError) { - ReportError(id_tok, "expected identifier after '.'"); + ReportSyntaxError(id_tok, "expected identifier after '.'"); } SynchronizeOnDelimiter(); return; @@ -643,7 +644,7 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { if (id_tok.type != TokenType::kIdent && id_tok.type != TokenType::kReservedWord) { if (id_tok.type != TokenType::kError) { - ReportError(id_tok, "expected identifier"); + ReportSyntaxError(id_tok, "expected identifier"); } return ast_factory_.NewUnspecified(NextId(id_tok)); } @@ -655,16 +656,17 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { } std::string name = leading_dot ? absl::StrCat(".", id_text) : std::string(id_text); - int64_t id = NextId(leading_dot ? first_tok : id_tok); if (peek_token_.type == TokenType::kLeftParen) { - NextToken(); + Token lparen = NextToken(); + int64_t call_id = NextId(lparen); std::vector args = ParseArguments(TokenType::kRightParen); - if (auto expanded = TryExpandMacro(id, name, nullptr, args); + if (auto expanded = TryExpandMacro(call_id, name, nullptr, args); expanded.has_value()) { return std::move(*expanded); } - return ast_factory_.NewCall(id, name, std::move(args)); + return ast_factory_.NewCall(call_id, name, std::move(args)); } + int64_t id = NextId(leading_dot ? first_tok : id_tok); return ast_factory_.NewIdent(id, std::move(name)); } @@ -717,11 +719,10 @@ ExprNode PrattParserWorker::ParsePrimary() { Token bad_tok = NextToken(); if (bad_tok.type != TokenType::kError) { if (bad_tok.type == TokenType::kEnd) { - ReportError( - bad_tok, - "Syntax error: mismatched input '' expecting expression"); + ReportSyntaxError(bad_tok, + "mismatched input '' expecting expression"); } else { - ReportError(bad_tok, "unexpected token"); + ReportSyntaxError(bad_tok, "unexpected token"); } } return ast_factory_.NewUnspecified(NextId(bad_tok)); @@ -815,7 +816,7 @@ ExprNode PrattParserWorker::ParseStruct( Token field_tok = NextToken(); if (field_tok.type != TokenType::kIdent && field_tok.type != TokenType::kReservedWord) { - ReportError(field_tok, "expected struct field name"); + ReportSyntaxError(field_tok, "expected struct field name"); SynchronizeOnDelimiter(); break; } @@ -879,7 +880,7 @@ ExprNode PrattParserWorker::ParseIntLiteral() { } else if (absl::SimpleAtoi(value, &int_val)) { return ast_factory_.NewIntConst(NextId(tok), int_val); } - ReportError(tok, "invalid int literal"); + ReportSyntaxError(tok, "invalid int literal"); return ast_factory_.NewUnspecified(NextId(tok)); } @@ -913,7 +914,7 @@ ExprNode PrattParserWorker::ParseNegativeIntLiteral(int64_t node_id) { if (success) { return ast_factory_.NewIntConst(node_id, int_val); } - ReportError(lit_tok, "invalid int literal"); + ReportSyntaxError(lit_tok, "invalid int literal"); return ast_factory_.NewUnspecified(NextId(lit_tok)); } @@ -933,7 +934,7 @@ ExprNode PrattParserWorker::ParseUintLiteral() { } else if (absl::SimpleAtoi(value, &uint_val)) { return ast_factory_.NewUintConst(NextId(tok), uint_val); } - ReportError(tok, "invalid uint literal"); + ReportSyntaxError(tok, "invalid uint literal"); return ast_factory_.NewUnspecified(NextId(tok)); } @@ -946,7 +947,7 @@ ExprNode PrattParserWorker::ParseDoubleLiteral() { if (absl::SimpleAtod(value, &double_val)) { return ast_factory_.NewDoubleConst(NextId(tok), double_val); } - ReportError(tok, "invalid double literal"); + ReportSyntaxError(tok, "invalid double literal"); return ast_factory_.NewUnspecified(NextId(tok)); } @@ -958,7 +959,7 @@ ExprNode PrattParserWorker::ParseNegativeDoubleLiteral( if (absl::SimpleAtod(GetTokenText(lit_tok), &double_val)) { return ast_factory_.NewDoubleConst(node_id, -double_val); } - ReportError(lit_tok, "invalid double literal"); + ReportSyntaxError(lit_tok, "invalid double literal"); return ast_factory_.NewUnspecified(NextId(lit_tok)); } diff --git a/parser/parser_test.cc b/parser/parser_test.cc index dbac2b02f..cc9fbc5fe 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -212,7 +212,7 @@ std::vector test_cases = { " | .^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:2: expected '}'\n" + "ERROR: :1:2: Syntax error: expected '}'\n" " | {\n" " | .^"}, @@ -480,10 +480,10 @@ std::vector test_cases = { " | ......^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:1: unexpected token\n" + "ERROR: :1:1: Syntax error: unexpected token\n" " | *@a | b\n" " | ^\n" - "ERROR: :1:2: unexpected character\n" + "ERROR: :1:2: Syntax error: unexpected character\n" " | *@a | b\n" " | .^"}, {"a | b", "", @@ -495,7 +495,7 @@ std::vector test_cases = { " | ....^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:3: unexpected single '|', expected '||'\n" + "ERROR: :1:3: Syntax error: unexpected single '|', expected '||'\n" " | a | b\n" " | ..^"}, {"?", "", @@ -508,7 +508,7 @@ std::vector test_cases = { "ERROR: :4294967295:0: <> parsetree", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:1: unexpected token\n" + "ERROR: :1:1: Syntax error: unexpected token\n" " | ?\n" " | ^"}, {"t{>C}", "", @@ -519,7 +519,7 @@ std::vector test_cases = { "mismatched input '}' expecting ':'\n | t{>C}\n | ....^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:3: expected struct field name\n" + "ERROR: :1:3: Syntax error: expected struct field name\n" " | t{>C}\n" " | ..^"}, {"foo(a,b,)", "", @@ -696,7 +696,7 @@ std::vector test_cases = { " | ..............^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:15: unexpected token after expression\n" + "ERROR: :1:15: Syntax error: unexpected token after expression\n" " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" " | ..............^"}, {"size(x) == x.size()", @@ -718,7 +718,7 @@ std::vector test_cases = { " | .....^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:5: unexpected character\n" + "ERROR: :1:5: Syntax error: unexpected character\n" " | 1 + $\n" " | ....^"}, {"1 + 2\n" @@ -729,7 +729,7 @@ std::vector test_cases = { " | ^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :2:1: unexpected token after expression\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" " | 3 +\n" " | ^"}, {"\"\\\"\"", "\"\\\"\"^#1:string#"}, @@ -870,7 +870,7 @@ std::vector test_cases = { " | .....^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:5: unexpected token\n" + "ERROR: :1:5: Syntax error: unexpected token\n" " | 1 + +\n" " | ....^"}, {"\"abc\" + \"def\"", @@ -885,7 +885,7 @@ std::vector test_cases = { " | .........^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:10: expected identifier after '.'\n" + "ERROR: :1:10: Syntax error: expected identifier after '.'\n" " | {\"a\": 1}.\"a\"\n" " | .........^"}, {"\"\\xC3\\XBF\"", "\"ÿ\"^#1:string#"}, @@ -985,10 +985,10 @@ std::vector test_cases = { " | ..........^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :2:7: unexpected token\n" + "ERROR: :2:7: Syntax error: unexpected token\n" " | && in.😁\n" " | ......^\n" - "ERROR: :2:10: unexpected character\n" + "ERROR: :2:10: Syntax error: unexpected character\n" " | && in.😁\n" " | .........^"}, {"as", "", @@ -1041,7 +1041,7 @@ std::vector test_cases = { " | ..^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:1: unexpected token\n" + "ERROR: :1:1: Syntax error: unexpected token\n" " | in\n" " | ^"}, {"let", "", @@ -1129,7 +1129,7 @@ std::vector test_cases = { " | ..^", "", "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :6:3: expected ']'\n" + "ERROR: :6:3: Syntax error: expected ']'\n" " | \r\n" " | ..^"}, From 2d4e7a2a3d8caaa6a4b17288d2d7f3f646ec7b9e Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 31 Aug 2026 15:40:24 -0700 Subject: [PATCH 206/350] [Pratt parser] Align Pratt parser AST node ID generation with the standard parser. PiperOrigin-RevId: 974116830 --- parser/internal/pratt_parser_test.cc | 16 ++-- parser/internal/pratt_parser_worker.cc | 8 ++ parser/internal/pratt_parser_worker.h | 93 ++++++++++--------- parser/parser_test.cc | 120 +++++++++---------------- 4 files changed, 108 insertions(+), 129 deletions(-) diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 422a5d19f..4fd2b02ec 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -434,7 +434,7 @@ std::vector GetParserTestCases() { TestCase{ .source = "- -1", .expected_ast = R"( - 1^#3:int64# + 1^#1:int64# )", }, TestCase{ @@ -452,7 +452,7 @@ std::vector GetParserTestCases() { .source = "---a", .expected_ast = R"( -_( - a^#4:Expr.Ident# + a^#2:Expr.Ident# )^#1:Expr.Call# )", }, @@ -1082,8 +1082,8 @@ std::vector GetParserTestCases() { .source = "{'key': 'value', 'num': 42}", .expected_ast = R"( { - "key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, - "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + "key"^#3:string#:"value"^#4:string#^#2:Expr.CreateStruct.Entry#, + "num"^#6:string#:42^#7:int64#^#5:Expr.CreateStruct.Entry# }^#1:Expr.CreateMap# )", }, @@ -1091,8 +1091,8 @@ std::vector GetParserTestCases() { .source = "{?'key': 'value', 'num': 42}", .expected_ast = R"( { - ?"key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, - "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + ?"key"^#3:string#:"value"^#4:string#^#2:Expr.CreateStruct.Entry#, + "num"^#6:string#:42^#7:int64#^#5:Expr.CreateStruct.Entry# }^#1:Expr.CreateMap# )", .enable_optional_syntax = true, @@ -1101,8 +1101,8 @@ std::vector GetParserTestCases() { .source = "{foo: 5, bar: \"xyz\"}", .expected_ast = R"( { - foo^#2:Expr.Ident#:5^#4:int64#^#3:Expr.CreateStruct.Entry#, - bar^#5:Expr.Ident#:"xyz"^#7:string#^#6:Expr.CreateStruct.Entry# + foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#, + bar^#6:Expr.Ident#:"xyz"^#7:string#^#5:Expr.CreateStruct.Entry# }^#1:Expr.CreateMap# )", }, diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index 198a70c4f..b01408774 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -216,6 +216,14 @@ int64_t ParserWorker::NextId(int32_t position) { int64_t ParserWorker::NextId() { return NextId(-1); } +void ParserWorker::SetPosition(int64_t id, const Token& token) { + if (token.start >= 0) { + positions_[id] = token.start; + SetNodeRange(id, token.start, + token.end > token.start ? token.end - 1 : token.start); + } +} + int64_t ParserWorker::CopyId(int64_t id) { if (id == 0) { return 0; diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 966717af1..e1608a3a2 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -16,6 +16,7 @@ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ #include +#include #include #include #include @@ -83,14 +84,12 @@ class ParserWorker { int64_t NextId(int32_t position); int64_t NextId(const Token& token) { int64_t id = NextId(token.start); - if (ABSL_PREDICT_FALSE(track_node_ranges_)) { - if (token.start >= 0 && token.end > token.start) { - node_ranges_[id] = {token.start, token.end - 1}; - } - } + SetNodeRange(id, token.start, + token.end > token.start ? token.end - 1 : token.start); return id; } int64_t NextId(); + void SetPosition(int64_t id, const Token& token); int64_t CopyId(int64_t id); void EraseId(int64_t id); void SetNodeRange(int64_t id, int32_t begin, int32_t end) { @@ -150,6 +149,7 @@ template class PrattParserWorker : public ParserWorker { public: using ParserWorker::NextId; + using ParserWorker::SetPosition; explicit PrattParserWorker( const cel::Source& source, const cel::ParserOptions& options, @@ -530,57 +530,61 @@ void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { template ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { - struct UnaryOpInfo { - TokenType type; - int64_t id; + struct UnaryOp { + Token token; + int64_t id = 0; }; - std::vector ops; - ops.push_back({first_op.type, NextId(first_op)}); - + std::vector ops; + ops.push_back({first_op}); while (peek_token_.type == TokenType::kExclamation || peek_token_.type == TokenType::kMinus) { - Token op = NextToken(); - ops.push_back({op.type, NextId(op)}); + ops.push_back({NextToken()}); } - ExprNode operand; - if (!ops.empty() && ops.back().type == TokenType::kMinus) { - if (options_.fold_unary_operators && ops.size() > 1 && - ops[ops.size() - 2].type == TokenType::kMinus) { - // Match the ANTLR parser behavior where `-(-)+` prefers to match as - // repeated negate operators instead of a negation of an int literal. - // ---9223372036854775808 will fail to parse. - ops.pop_back(); - ops.pop_back(); - operand = ParseSelectorChain(); - } else if (peek_token_.type == TokenType::kInt) { - int64_t op_id = ops.back().id; - ops.pop_back(); - operand = ParseNegativeIntLiteral(op_id); - ParseSelectorChainTail(operand); - } else if (peek_token_.type == TokenType::kFloat) { - int64_t op_id = ops.back().id; - ops.pop_back(); - operand = ParseNegativeDoubleLiteral(op_id); - ParseSelectorChainTail(operand); - } else { - operand = ParseSelectorChain(); + const bool has_solitary_trailing_minus = + !ops.empty() && ops.back().token.type == TokenType::kMinus && + (ops.size() == 1 || ops[ops.size() - 2].token.type != TokenType::kMinus); + + if (options_.fold_unary_operators) { + size_t write = 0; + for (size_t read = 0; read < ops.size();) { + size_t next = read; + while (next < ops.size() && + ops[next].token.type == ops[read].token.type) { + next++; + } + if ((next - read) % 2 != 0) { + ops[write++] = ops[read]; + } + read = next; } + ops.resize(write); + } + + for (auto& op : ops) { + op.id = NextId(op.token); + } + + ExprNode operand; + // Match the ANTLR parser behavior where `-(-)+` prefers to match as + // repeated negate operators instead of a negation of an int literal. + // ---9223372036854775808 will fail to parse. + if (has_solitary_trailing_minus && (peek_token_.type == TokenType::kInt || + peek_token_.type == TokenType::kFloat)) { + int64_t op_id = ops.back().id; + ops.pop_back(); + operand = (peek_token_.type == TokenType::kInt) + ? ParseNegativeIntLiteral(op_id) + : ParseNegativeDoubleLiteral(op_id); + ParseSelectorChainTail(operand); } else { operand = ParseSelectorChain(); } for (int i = static_cast(ops.size()) - 1; i >= 0; --i) { std::vector args; - if (options_.fold_unary_operators && i > 0) { - if (ops[i - 1].type == ops[i].type) { - i--; - continue; - } - } - args.push_back(std::move(operand)); - absl::string_view op_name = (ops[i].type == TokenType::kExclamation) + absl::string_view op_name = (ops[i].token.type == TokenType::kExclamation) ? CelOperator::LOGICAL_NOT : CelOperator::NEGATE; operand = @@ -777,12 +781,13 @@ ExprNode PrattParserWorker::ParseMap() { } key_start = peek_token_; } + int64_t entry_id = NextId(); ExprNode key = ParseExpr(); Token colon = peek_token_; if (!Expect(TokenType::kColon, "expected ':' in map entry")) { break; } - int64_t entry_id = NextId(colon); + SetPosition(entry_id, colon); builder.Add(entry_id, std::move(key), ParseExpr(), optional); if (peek_token_.type == TokenType::kComma) { NextToken(); diff --git a/parser/parser_test.cc b/parser/parser_test.cc index cc9fbc5fe..d51339796 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -65,15 +65,8 @@ struct TestInfo { TestInfo(const std::string& I, const std::string& P, const std::string& E = "", const std::string& L = "", const std::string& R = "", const std::string& M = "", - const std::string& P_PRATT = "", const std::string& E_PRATT = "") - : I(I), - P(P), - E(E), - L(L), - R(R), - M(M), - P_PRATT(P_PRATT), - E_PRATT(E_PRATT) {} + const std::string& E_PRATT = "") + : I(I), P(P), E(E), L(L), R(R), M(M), E_PRATT(E_PRATT) {} // I contains the input expression to be parsed. std::string I; @@ -94,9 +87,6 @@ struct TestInfo { // M contains the expected macro call output of hte expression tree. std::string M; - // P_PRATT contains alternative adorned AST string when using pratt parser. - std::string P_PRATT; - // E_PRATT contains alternative error output when using pratt parser. std::string E_PRATT; }; @@ -173,12 +163,16 @@ std::vector test_cases = { "{\n" " foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" " bar^#6:Expr.Ident#:\"xyz\"^#7:string#^#5:Expr.CreateStruct.Entry#\n" - "}^#1:Expr.CreateStruct#", - "", "", "", "", - // PRATT PARSER AST + "}^#1:Expr.CreateStruct#"}, + {"{\"foo\": 5, \"bar\": \"xyz\"}", + "{\n" + " \"foo\"^#3:string#:5^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" + " \"bar\"^#6:string#:\"xyz\"^#7:string#^#5:Expr.CreateStruct.Entry#\n" + "}^#1:Expr.CreateStruct#"}, + {"{'a': 1, 'b': 2}", "{\n" - " foo^#2:Expr.Ident#:5^#4:int64#^#3:Expr.CreateStruct.Entry#,\n" - " bar^#5:Expr.Ident#:\"xyz\"^#7:string#^#6:Expr.CreateStruct.Entry#\n" + " \"a\"^#3:string#:1^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" + " \"b\"^#6:string#:2^#7:int64#^#5:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"a > 5 && a < 10", "_&&_(\n" @@ -210,7 +204,7 @@ std::vector test_cases = { "NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n | {\n" " | .^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:2: Syntax error: expected '}'\n" " | {\n" @@ -407,12 +401,6 @@ std::vector test_cases = { "{\n" " a^#3:Expr.Ident#:b^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry#,\n" " c^#6:Expr.Ident#:d^#7:Expr.Ident#^#5:Expr.CreateStruct.Entry#\n" - "}^#1:Expr.CreateStruct#", - "", "", "", "", - // PRATT PARSER AST - "{\n" - " a^#2:Expr.Ident#:b^#4:Expr.Ident#^#3:Expr.CreateStruct.Entry#,\n" - " c^#5:Expr.Ident#:d^#7:Expr.Ident#^#6:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"[]", "[]^#1:Expr.CreateList#"}, {"[a]", @@ -478,7 +466,7 @@ std::vector test_cases = { "ERROR: :1:7: Syntax error: extraneous input 'b' expecting \n" " | *@a | b\n" " | ......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: Syntax error: unexpected token\n" " | *@a | b\n" @@ -493,7 +481,7 @@ std::vector test_cases = { "ERROR: :1:5: Syntax error: extraneous input 'b' expecting \n" " | a | b\n" " | ....^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:3: Syntax error: unexpected single '|', expected '||'\n" " | a | b\n" @@ -506,7 +494,7 @@ std::vector test_cases = { "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n | ?\n | .^\n" "ERROR: :4294967295:0: <> parsetree", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: Syntax error: unexpected token\n" " | ?\n" @@ -517,7 +505,7 @@ std::vector test_cases = { ":1:5: " "Syntax error: " "mismatched input '}' expecting ':'\n | t{>C}\n | ....^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:3: Syntax error: expected struct field name\n" " | t{>C}\n" @@ -528,7 +516,7 @@ std::vector test_cases = { "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | foo(a,b,)\n" " | ........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:9: unexpected token\n" " | foo(a,b,)\n" @@ -678,12 +666,6 @@ std::vector test_cases = { "{\n" " 1^#3:int64#:2u^#4:uint64#^#2:Expr.CreateStruct.Entry#,\n" " 2^#6:int64#:3u^#7:uint64#^#5:Expr.CreateStruct.Entry#\n" - "}^#1:Expr.CreateStruct#", - "", "", "", "", - // PRATT PARSER AST - "{\n" - " 1^#2:int64#:2u^#4:uint64#^#3:Expr.CreateStruct.Entry#,\n" - " 2^#5:int64#:3u^#7:uint64#^#6:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"TestAllTypes{single_int32: 1, single_int64: 2}", "TestAllTypes{\n" @@ -694,7 +676,7 @@ std::vector test_cases = { "ERROR: :1:15: Syntax error: mismatched input '{' expecting \n" " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" " | ..............^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:15: Syntax error: unexpected token after expression\n" " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" @@ -716,7 +698,7 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | 1 + $\n" " | .....^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:5: Syntax error: unexpected character\n" " | 1 + $\n" @@ -727,7 +709,7 @@ std::vector test_cases = { "ERROR: :2:1: Syntax error: mismatched input '3' expecting \n" " | 3 +\n" " | ^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :2:1: Syntax error: unexpected token after expression\n" " | 3 +\n" @@ -750,7 +732,7 @@ std::vector test_cases = { "ERROR: :1:9: all() variable name must be a simple identifier\n" " | [].all(.x, x)\n" " | ........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:8: all() variable name must be a simple identifier\n" " | [].all(.x, x)\n" @@ -759,7 +741,7 @@ std::vector test_cases = { "ERROR: :1:12: exists() variable name must be a simple identifier\n" " | [].exists(.x, x)\n" " | ...........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:11: exists() variable name must be a simple identifier\n" " | [].exists(.x, x)\n" @@ -769,7 +751,7 @@ std::vector test_cases = { "identifier\n" " | [].exists_one(.x, x)\n" " | ...............^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:15: exists_one() variable name must be a simple " "identifier\n" @@ -779,7 +761,7 @@ std::vector test_cases = { "ERROR: :1:9: map() variable name must be a simple identifier\n" " | [].map(.x, x, x)\n" " | ........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:8: map() variable name must be a simple identifier\n" " | [].map(.x, x, x)\n" @@ -788,7 +770,7 @@ std::vector test_cases = { "ERROR: :1:12: filter() variable name must be a simple identifier\n" " | [].filter(.x, x)\n" " | ...........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:11: filter() variable name must be a simple identifier\n" " | [].filter(.x, x)\n" @@ -848,11 +830,6 @@ std::vector test_cases = { {"---a", "-_(\n" " a^#2:Expr.Ident#\n" - ")^#1:Expr.Call#", - "", "", "", "", - // PRATT PARSER AST - "-_(\n" - " a^#4:Expr.Ident#\n" ")^#1:Expr.Call#"}, {"1 + +", "", "ERROR: :1:5: Syntax error: mismatched input '+' expecting {'[', " @@ -868,7 +845,7 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | 1 + +\n" " | .....^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:5: Syntax error: unexpected token\n" " | 1 + +\n" @@ -883,7 +860,7 @@ std::vector test_cases = { "'.\"a\"'\n" " | {\"a\": 1}.\"a\"\n" " | .........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:10: Syntax error: expected identifier after '.'\n" " | {\"a\": 1}.\"a\"\n" @@ -907,7 +884,7 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | \"\\xFh\"\n" " | ......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: Invalid string literal: Illegal escape sequence: Hex " "escape must be followed by 2 hex digits but saw: \\xFh\n" @@ -927,7 +904,7 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | \"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\"\n" " | ..........................................^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: Invalid string literal: Illegal escape sequence: " "\\>\n" @@ -983,7 +960,7 @@ std::vector test_cases = { "ERROR: :2:11: Syntax error: no viable alternative at input '.'\n" " | && in.😁\n" " | ..........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :2:7: Syntax error: unexpected token\n" " | && in.😁\n" @@ -1039,7 +1016,7 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | in\n" " | ..^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: Syntax error: unexpected token\n" " | in\n" @@ -1089,7 +1066,7 @@ std::vector test_cases = { "ERROR: :1:26: reserved identifier: var\n" " | [1, 2, 3].map(var, var * var)\n" " | .........................^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:15: reserved identifier: var\n" " | [1, 2, 3].map(var, var * var)\n" @@ -1127,7 +1104,7 @@ std::vector test_cases = { "{']', ','}\n" " | \r\n" " | ..^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :6:3: Syntax error: expected ']'\n" " | \r\n" @@ -1173,7 +1150,7 @@ std::vector test_cases = { "ERROR: :1:7: Syntax error: token recognition error at: '`'\n" " | a.`b c`\n" " | ......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:3: unexpected quoted identifier\n" " | a.`b c`\n" @@ -1185,7 +1162,7 @@ std::vector test_cases = { "ERROR: :1:8: Syntax error: token recognition error at: '`'\n" " | a.`@foo`\n" " | .......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:3: unexpected quoted identifier\n" " | a.`@foo`\n" @@ -1197,7 +1174,7 @@ std::vector test_cases = { "ERROR: :1:8: Syntax error: token recognition error at: '`'\n" " | a.`$foo`\n" " | .......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:3: unexpected quoted identifier\n" " | a.`$foo`\n" @@ -1210,7 +1187,7 @@ std::vector test_cases = { "BYTES, IDENTIFIER}\n" " | `a.b`\n" " | ^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: unexpected quoted identifier\n" " | `a.b`\n" @@ -1226,7 +1203,7 @@ std::vector test_cases = { "_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | `a.b`()\n" " | ......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: unexpected quoted identifier\n" " | `a.b`()\n" @@ -1235,7 +1212,7 @@ std::vector test_cases = { "ERROR: :1:10: Syntax error: mismatched input '(' expecting \n" " | foo.`a.b`()\n" " | .........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:5: unexpected quoted identifier\n" " | foo.`a.b`()\n" @@ -1292,8 +1269,7 @@ std::vector test_cases = { " @result^#24:Expr.Ident#\n" " )^#25:Expr.Call#,\n" " // Result\n" - " @result^#26:Expr.Ident#)^#27:Expr.Comprehension#" - "", + " @result^#26:Expr.Ident#)^#27:Expr.Comprehension#", "", "", "", "x^#1:Expr.Ident#.filter(\n" " y^#3:Expr.Ident#,\n" @@ -1515,12 +1491,7 @@ std::vector test_cases = { {"{?'key': value}", "{\n " "?\"key\"^#3:string#:value^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry#\n}^#" - "1:Expr.CreateStruct#", - "", "", "", "", - // PRATT PARSER AST - "{\n" - " ?\"key\"^#2:string#:value^#4:Expr.Ident#^#3:Expr.CreateStruct.Entry#\n" - "}^#1:Expr.CreateStruct#"}, + "1:Expr.CreateStruct#"}, {"[?a, ?b]", "[\n ?a^#2:Expr.Ident#,\n ?b^#3:Expr.Ident#\n]^#1:Expr.CreateList#"}, {"[?a[?b]]", @@ -1759,13 +1730,8 @@ TEST_P(ExpressionTest, Parse) { KindAndIdAdorner kind_and_id_adorner; ExprPrinter w(kind_and_id_adorner); std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); - if (options_.enable_pratt_parser && !test_info.P_PRATT.empty()) { - EXPECT_EQ(test_info.P_PRATT, adorned_string) - << result->parsed_expr().ShortDebugString(); - } else { - EXPECT_EQ(test_info.P, adorned_string) - << result->parsed_expr().ShortDebugString(); - } + EXPECT_EQ(test_info.P, adorned_string) + << result->parsed_expr().ShortDebugString(); } if (!options_.enable_pratt_parser && !test_info.L.empty()) { From 474f1af3275be8baa129da07bdabd30dc44aea33 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 31 Aug 2026 16:38:58 -0700 Subject: [PATCH 207/350] Update field access behavior to tolerate out of range timestamps. Preserves existing behavior of accepting an out of range value if it comes from proto. PiperOrigin-RevId: 974144329 --- common/legacy_value.h | 8 -- common/value.cc | 7 +- common/values/duration_value.h | 6 ++ common/values/struct_value_builder.cc | 22 ++--- common/values/timestamp_value.h | 6 ++ internal/message_equality.cc | 4 +- internal/well_known_types.cc | 6 +- internal/well_known_types_test.cc | 93 +++++++++++++++++-- runtime/internal/function_adapter.h | 4 +- runtime/standard/type_conversion_functions.cc | 6 +- 10 files changed, 123 insertions(+), 39 deletions(-) diff --git a/common/legacy_value.h b/common/legacy_value.h index e71eb85a7..8fea22ec0 100644 --- a/common/legacy_value.h +++ b/common/legacy_value.h @@ -121,14 +121,6 @@ inline MapValue CreateLegacyMapValue( return common_internal::LegacyMapValue(value); } -inline Value CreateDurationValue(absl::Duration value, bool unchecked = false) { - return DurationValue{value}; -} - -inline TimestampValue CreateTimestampValue(absl::Time value) { - return TimestampValue{value}; -} - Value LegacyValueToModernValueOrDie( google::protobuf::Arena* arena, const google::api::expr::runtime::CelValue& value, bool unchecked = false); diff --git a/common/value.cc b/common/value.cc index e749a16c6..a2ffea620 100644 --- a/common/value.cc +++ b/common/value.cc @@ -1146,10 +1146,13 @@ Value VistWellKnownTypeValue(float value) { return DoubleValue(value); } Value VistWellKnownTypeValue(double value) { return DoubleValue(value); } Value VistWellKnownTypeValue(absl::Duration value) { - return DurationValue(value); + // Tolerate out-of-range values. + return UnsafeDurationValue(value); } -Value VistWellKnownTypeValue(absl::Time value) { return TimestampValue(value); } +Value VistWellKnownTypeValue(absl::Time value) { + return UnsafeTimestampValue(value); +} struct OwningWellKnownTypesValueVisitor { google::protobuf::Arena* absl_nullable arena; diff --git a/common/values/duration_value.h b/common/values/duration_value.h index 1b2468b60..0c934f097 100644 --- a/common/values/duration_value.h +++ b/common/values/duration_value.h @@ -51,6 +51,12 @@ class DurationValue final : private common_internal::ValueMixin { public: static constexpr ValueKind kKind = ValueKind::kDuration; + // Constructs a `DurationValue` from an `absl::Duration`. + // + // DCHECK-fails if the value is not in the supported range. + // + // Prefer using `SafeDurationValue` or `UnsafeDurationValue` if the caller + // has already validated the value. explicit DurationValue(absl::Duration value) noexcept : DurationValue(absl::in_place, value) { ABSL_DCHECK_OK(internal::ValidateDuration(value)); diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index d5206acbf..f43369e33 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -254,8 +254,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( if (auto duration_value = value.AsDuration(); duration_value) { CEL_RETURN_IF_ERROR( well_known_types->Duration().Initialize(message->GetDescriptor())); - CEL_RETURN_IF_ERROR(well_known_types->Duration().SetFromAbslDuration( - message, duration_value->NativeValue())); + well_known_types->Duration().UnsafeSetFromAbslDuration( + message, duration_value->NativeValue()); return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); @@ -264,8 +264,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( if (auto timestamp_value = value.AsTimestamp(); timestamp_value) { CEL_RETURN_IF_ERROR( well_known_types->Timestamp().Initialize(message->GetDescriptor())); - CEL_RETURN_IF_ERROR(well_known_types->Timestamp().SetFromAbslTime( - message, timestamp_value->NativeValue())); + well_known_types->Timestamp().UnsafeSetFromAbslTime( + message, timestamp_value->NativeValue()); return std::nullopt; } return TypeConversionError(value.GetTypeName(), to_desc->full_name()); @@ -1304,11 +1304,11 @@ class MessageValueBuilderImpl { if (auto duration_value = value.AsDuration(); duration_value) { CEL_RETURN_IF_ERROR(well_known_types_.Duration().Initialize( field->message_type())); - CEL_RETURN_IF_ERROR( - well_known_types_.Duration().SetFromAbslDuration( - reflection_->MutableMessage(message_, field, - message_factory_), - duration_value->NativeValue())); + + well_known_types_.Duration().UnsafeSetFromAbslDuration( + reflection_->MutableMessage(message_, field, + message_factory_), + duration_value->NativeValue()); return std::nullopt; } return TypeConversionError(value.GetTypeName(), @@ -1322,10 +1322,10 @@ class MessageValueBuilderImpl { if (auto timestamp_value = value.AsTimestamp(); timestamp_value) { CEL_RETURN_IF_ERROR(well_known_types_.Timestamp().Initialize( field->message_type())); - CEL_RETURN_IF_ERROR(well_known_types_.Timestamp().SetFromAbslTime( + well_known_types_.Timestamp().UnsafeSetFromAbslTime( reflection_->MutableMessage(message_, field, message_factory_), - timestamp_value->NativeValue())); + timestamp_value->NativeValue()); return std::nullopt; } return TypeConversionError(value.GetTypeName(), diff --git a/common/values/timestamp_value.h b/common/values/timestamp_value.h index acc202300..87eae0ed0 100644 --- a/common/values/timestamp_value.h +++ b/common/values/timestamp_value.h @@ -52,6 +52,12 @@ class TimestampValue final public: static constexpr ValueKind kKind = ValueKind::kTimestamp; + // Constructs a `TimestampValue` from an `absl::Time`. + // + // DCHECK-fails if the value is not in the supported range. + // + // Prefer using `SafeTimestampValue` or `UnsafeTimestampValue` if the caller + // has already validated the value. explicit TimestampValue(absl::Time value) noexcept : TimestampValue(absl::in_place, value) { ABSL_DCHECK_OK(internal::ValidateTimestamp(value)); diff --git a/internal/message_equality.cc b/internal/message_equality.cc index 33ef78089..34080e6b1 100644 --- a/internal/message_equality.cc +++ b/internal/message_equality.cc @@ -380,11 +380,11 @@ absl::StatusOr AsEquatableValue( case Descriptor::WELLKNOWNTYPE_DURATION: CEL_RETURN_IF_ERROR( reflection.duration_reflection.Initialize(descriptor)); - return reflection.duration_reflection.ToAbslDuration(message); + return reflection.duration_reflection.UnsafeToAbslDuration(message); case Descriptor::WELLKNOWNTYPE_TIMESTAMP: CEL_RETURN_IF_ERROR( reflection.timestamp_reflection.Initialize(descriptor)); - return reflection.timestamp_reflection.ToAbslTime(message); + return reflection.timestamp_reflection.UnsafeToAbslTime(message); case Descriptor::WELLKNOWNTYPE_ANY: return EquatableAny(message); default: diff --git a/internal/well_known_types.cc b/internal/well_known_types.cc index 02e50c3e3..175f978d4 100644 --- a/internal/well_known_types.cc +++ b/internal/well_known_types.cc @@ -2114,13 +2114,15 @@ absl::StatusOr AdaptFromMessage( case Descriptor::WELLKNOWNTYPE_ANY: // This is unreachable, as AdaptAny() above recursively unpacks. ABSL_UNREACHABLE(); + // Don't check that time values are in range on field access. Assume + // error will propagate if they are used in any arithmetic. case Descriptor::WELLKNOWNTYPE_DURATION: { CEL_ASSIGN_OR_RETURN(auto reflection, GetDurationReflection(descriptor)); - return reflection.ToAbslDuration(*to_adapt); + return reflection.UnsafeToAbslDuration(*to_adapt); } case Descriptor::WELLKNOWNTYPE_TIMESTAMP: { CEL_ASSIGN_OR_RETURN(auto reflection, GetTimestampReflection(descriptor)); - return reflection.ToAbslTime(*to_adapt); + return reflection.UnsafeToAbslTime(*to_adapt); } case Descriptor::WELLKNOWNTYPE_VALUE: { CEL_ASSIGN_OR_RETURN(auto reflection, GetValueReflection(descriptor)); diff --git a/internal/well_known_types_test.cc b/internal/well_known_types_test.cc index afc8ce396..5ffc06ee9 100644 --- a/internal/well_known_types_test.cc +++ b/internal/well_known_types_test.cc @@ -341,6 +341,44 @@ TEST_F(ReflectionTest, Duration_Dynamic) { StatusIs(absl::StatusCode::kInvalidArgument)); } +TEST_F(ReflectionTest, Duration_ToAbslDuration) { + auto* value = MakeDynamic(); + ASSERT_OK_AND_ASSIGN( + auto reflection, + GetDurationReflection(ABSL_DIE_IF_NULL(value->GetDescriptor()))); + + reflection.SetSeconds(value, 1); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslDuration(*value), + IsOkAndHolds(absl::Seconds(1) + absl::Nanoseconds(1))); + EXPECT_EQ(reflection.UnsafeToAbslDuration(*value), + absl::Seconds(1) + absl::Nanoseconds(1)); + + reflection.SetSeconds(value, 0x7fffffffffffffff); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslDuration(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid duration seconds: "))); + EXPECT_EQ(reflection.UnsafeToAbslDuration(*value), + absl::Seconds(0x7fffffffffffffff) + absl::Nanoseconds(1)); + + reflection.SetSeconds(value, 1); + reflection.SetNanos(value, 0x7fffffff); + EXPECT_THAT(reflection.ToAbslDuration(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid duration nanoseconds: "))); + EXPECT_EQ(reflection.UnsafeToAbslDuration(*value), + absl::Seconds(1) + absl::Nanoseconds(0x7fffffff)); + + reflection.SetSeconds(value, -1); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslDuration(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("duration sign mismatch: "))); + EXPECT_EQ(reflection.UnsafeToAbslDuration(*value), + absl::Seconds(-1) + absl::Nanoseconds(1)); +} + TEST_F(ReflectionTest, Timestamp_Generated) { auto* value = MakeGenerated(); EXPECT_EQ(TimestampReflection::GetSeconds(*value), 0); @@ -389,6 +427,39 @@ TEST_F(ReflectionTest, Timestamp_Dynamic) { StatusIs(absl::StatusCode::kInvalidArgument)); } +TEST_F(ReflectionTest, Timestamp_ToAbslTime) { + auto* value = MakeDynamic(); + ASSERT_OK_AND_ASSIGN( + auto reflection, + GetTimestampReflection(ABSL_DIE_IF_NULL(value->GetDescriptor()))); + + reflection.SetSeconds(value, 1); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslTime(*value), + IsOkAndHolds(absl::UnixEpoch() + absl::Seconds(1) + + absl::Nanoseconds(1))); + EXPECT_EQ(reflection.UnsafeToAbslTime(*value), + absl::UnixEpoch() + absl::Seconds(1) + absl::Nanoseconds(1)); + + reflection.SetSeconds(value, 0x7fffffffffffffff); + reflection.SetNanos(value, 1); + EXPECT_THAT(reflection.ToAbslTime(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid timestamp seconds: "))); + EXPECT_EQ(reflection.UnsafeToAbslTime(*value), + absl::UnixEpoch() + absl::Seconds(0x7fffffffffffffff) + + absl::Nanoseconds(1)); + + reflection.SetSeconds(value, 1); + reflection.SetNanos(value, 0x7fffffff); + EXPECT_THAT(reflection.ToAbslTime(*value), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("invalid timestamp nanoseconds: "))); + EXPECT_EQ( + reflection.UnsafeToAbslTime(*value), + absl::UnixEpoch() + absl::Seconds(1) + absl::Nanoseconds(0x7fffffff)); +} + TEST_F(ReflectionTest, Value_Generated) { auto* value = MakeGenerated(); EXPECT_EQ(ValueReflection::GetKindCase(*value), @@ -698,16 +769,16 @@ TEST_F(AdaptFromMessageTest, Duration_SecondsOutOfRange) { auto message = DynamicParseTextProto( R"pb(seconds: 0x7fffffffffffffff nanos: 1)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("invalid duration seconds: "))); + IsOkAndHolds(VariantWith( + absl::Seconds(0x7fffffffffffffff) + absl::Nanoseconds(1)))); } TEST_F(AdaptFromMessageTest, Duration_NanosOutOfRange) { auto message = DynamicParseTextProto( R"pb(seconds: 1 nanos: 0x7fffffff)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("invalid duration nanoseconds: "))); + IsOkAndHolds(VariantWith( + absl::Seconds(1) + absl::Nanoseconds(0x7fffffff)))); } TEST_F(AdaptFromMessageTest, Duration_SignMismatch) { @@ -715,8 +786,8 @@ TEST_F(AdaptFromMessageTest, Duration_SignMismatch) { DynamicParseTextProto(R"pb(seconds: -1 nanos: 1)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("duration sign mismatch: "))); + IsOkAndHolds(VariantWith(absl::Seconds(-1) + + absl::Nanoseconds(1)))); } TEST_F(AdaptFromMessageTest, Timestamp) { @@ -733,16 +804,18 @@ TEST_F(AdaptFromMessageTest, Timestamp_SecondsOutOfRange) { auto message = DynamicParseTextProto( R"pb(seconds: 0x7fffffffffffffff nanos: 1)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("invalid timestamp seconds: "))); + IsOkAndHolds(VariantWith( + absl::UnixEpoch() + absl::Seconds(0x7fffffffffffffff) + + absl::Nanoseconds(1)))); } TEST_F(AdaptFromMessageTest, Timestamp_NanosOutOfRange) { auto message = DynamicParseTextProto( R"pb(seconds: 1 nanos: 0x7fffffff)pb"); EXPECT_THAT(AdaptFromMessage(*message), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("invalid timestamp nanoseconds: "))); + IsOkAndHolds( + VariantWith(absl::UnixEpoch() + absl::Seconds(1) + + absl::Nanoseconds(0x7fffffff)))); } TEST_F(AdaptFromMessageTest, Value_NullValue) { diff --git a/runtime/internal/function_adapter.h b/runtime/internal/function_adapter.h index 9b191e577..1ff497486 100644 --- a/runtime/internal/function_adapter.h +++ b/runtime/internal/function_adapter.h @@ -200,13 +200,13 @@ struct AdaptedToValueVisitor { absl::StatusOr operator()(absl::Time in) { // Type matching may have already occurred. It's too late to change up the // type and return an error. - return TimestampValue(in); + return UnsafeTimestampValue(in); } absl::StatusOr operator()(absl::Duration in) { // Type matching may have already occurred. It's too late to change up the // type and return an error. - return DurationValue(in); + return UnsafeDurationValue(in); } absl::StatusOr operator()(Value in) { return in; } diff --git a/runtime/standard/type_conversion_functions.cc b/runtime/standard/type_conversion_functions.cc index 76e95751b..2400c8fdf 100644 --- a/runtime/standard/type_conversion_functions.cc +++ b/runtime/standard/type_conversion_functions.cc @@ -403,14 +403,16 @@ absl::Status RegisterTimeConversionFunctions(FunctionRegistry& registry, CEL_RETURN_IF_ERROR( (UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kTimestamp, - [](absl::Time value) -> Value { return TimestampValue(value); }, + [](absl::Time value) -> Value { return UnsafeTimestampValue(value); }, registry))); // duration -> duration CEL_RETURN_IF_ERROR( (UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kDuration, - [](absl::Duration value) -> Value { return DurationValue(value); }, + [](absl::Duration value) -> Value { + return UnsafeDurationValue(value); + }, registry))); // timestamp() conversion from string. From c5805928f78b990ae2d223dd429fd0f9fd3a9d19 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 31 Aug 2026 17:40:21 -0700 Subject: [PATCH 208/350] Rollforward: Opt in to new field selection implementation. Was rolled back due to internal test affected by stricter timestamp validation. Those validations were relaxed. PiperOrigin-RevId: 974172394 --- eval/public/cel_options.h | 2 +- runtime/runtime_options.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index b0d9e6db0..a6e2692d4 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -232,7 +232,7 @@ struct InterpreterOptions { // For google::api::expr::runtime::CelExpression, this will enable updated // implementations for field access on protobuf messages, aligned with the // cel::Value implementation. - bool enable_use_new_field_select_implementation = false; + bool enable_use_new_field_select_implementation = true; }; // LINT.ThenChange(//depot/google3/runtime/runtime_options.h) diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 245b8d52c..831183dc8 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -210,7 +210,7 @@ struct RuntimeOptions { // For google::api::expr::runtime::CelExpression, this will enable updated // implementations for field access on protobuf messages, aligned with the // cel::Value implementation. - bool enable_use_new_field_select_implementation = false; + bool enable_use_new_field_select_implementation = true; }; // LINT.ThenChange(//depot/google3/eval/public/cel_options.h) From 04ffade164c494fd64ff16d9cac7496d14cbc08c Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 1 Sep 2026 12:15:34 -0700 Subject: [PATCH 209/350] Migrate gaer::CelProtoWrapper::CreateMessage to just call the modern equivalent. Add overload for specifying the expected message factory and descriptor pool. PiperOrigin-RevId: 974637296 --- eval/public/structs/BUILD | 9 +- eval/public/structs/cel_proto_wrapper.cc | 87 +++++++++++++++++-- eval/public/structs/cel_proto_wrapper.h | 21 ++++- eval/public/structs/cel_proto_wrapper_test.cc | 60 ++++++++++--- 4 files changed, 155 insertions(+), 22 deletions(-) diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 468867294..41c0d7047 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -30,10 +30,17 @@ cc_library( deps = [ ":cel_proto_wrap_util", ":proto_message_type_adapter", + ":trivial_legacy_type_info_internal", + "//common:value", "//eval/public:cel_value", "//eval/public:message_wrapper", "//internal:proto_time_encoding", - "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_protobuf//:duration_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:timestamp_cc_proto", diff --git a/eval/public/structs/cel_proto_wrapper.cc b/eval/public/structs/cel_proto_wrapper.cc index 6fad6aee3..e220b64dd 100644 --- a/eval/public/structs/cel_proto_wrapper.cc +++ b/eval/public/structs/cel_proto_wrapper.cc @@ -14,11 +14,21 @@ #include "eval/public/structs/cel_proto_wrapper.h" -#include "absl/types/optional.h" +#include + +#include "absl/base/no_destructor.h" +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "common/legacy_value.h" +#include "common/value.h" #include "eval/public/cel_value.h" #include "eval/public/message_wrapper.h" #include "eval/public/structs/cel_proto_wrap_util.h" #include "eval/public/structs/proto_message_type_adapter.h" +#include "eval/public/structs/trivial_legacy_type_info_internal.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -27,9 +37,36 @@ namespace google::api::expr::runtime { namespace { +using ::cel::interop_internal::TrivialTypeInfo; using ::google::protobuf::Arena; using ::google::protobuf::Descriptor; +using ::google::protobuf::DescriptorPool; using ::google::protobuf::Message; +using ::google::protobuf::MessageFactory; + +// Returns the arena for the given message, or the fallback arena if the +// message does not have an arena. +// +// A global fallback arena is used to avoid allocation when the arena is not +// specified. This is effectively a memory leak, but won't trigger leak +// check analyzers. +// +// This emulates the old behavior of tolerating a nullptr arena without +// triggering a crash. +google::protobuf::Arena* GetArena(const Message* absl_nonnull message, + google::protobuf::Arena* absl_nullable arena) { + if (arena != nullptr) { + return arena; + } + if (message->GetArena() != nullptr) { + return message->GetArena(); + } + static absl::NoDestructor fallback_arena; + ABSL_LOG(WARNING) + << "CelValue: using fallback global arena for wrapping message: " + << message->GetTypeName(); + return fallback_arena.get(); +} } // namespace @@ -38,14 +75,50 @@ CelValue CelProtoWrapper::InternalWrapMessage(const Message* message) { MessageWrapper(message, &GetGenericProtoTypeInfoInstance())); } -// CreateMessage creates CelValue from google::protobuf::Message. -// As some of CEL basic types are subclassing google::protobuf::Message, -// this method contains type checking and downcasts. -CelValue CelProtoWrapper::CreateMessage(const Message* value, Arena* arena) { - return internal::UnwrapMessageToValue(value, &InternalWrapMessage, arena); +CelValue CelProtoWrapper::CreateMessage( + const Message* absl_nonnull value, + const google::protobuf::DescriptorPool* absl_nonnull pool, + MessageFactory* absl_nonnull factory, Arena* absl_nonnull arena) { + ABSL_DCHECK(value != nullptr); + if (value->GetDescriptor() == nullptr || value->GetReflection() == nullptr) { + // This only happens for custom google::protobuf::Message subclasses that CEL can't + // support. + return CelValue::CreateMessageWrapper( + MessageWrapper(value, TrivialTypeInfo::GetInstance())); + } + + auto modern_value = + cel::Value::WrapMessageUnsafe(value, pool, factory, arena); + + absl::StatusOr cel_value = cel::LegacyValue(arena, modern_value); + if (!cel_value.ok()) { + // This only happens for custom google::protobuf::Message subclasses that CEL can't + // support. + auto* status = + google::protobuf::Arena::Create(arena, cel_value.status()); + return CelValue::CreateError(status); + } + return *cel_value; +} + +CelValue CelProtoWrapper::CreateMessage(const Message* absl_nullable value, + Arena* absl_nullable arena) { + if (value == nullptr) { + return CelValue::CreateNull(); + } + + if (value->GetDescriptor() == nullptr || value->GetReflection() == nullptr) { + // This only happens for custom messages subclasses that CEL can't support. + return CelValue::CreateMessageWrapper( + MessageWrapper(value, TrivialTypeInfo::GetInstance())); + } + const auto* pool = value->GetDescriptor()->file()->pool(); + auto* factory = value->GetReflection()->GetMessageFactory(); + arena = GetArena(value, arena); + return CreateMessage(value, pool, factory, arena); } -absl::optional CelProtoWrapper::MaybeWrapValue( +std::optional CelProtoWrapper::MaybeWrapValue( const Descriptor* descriptor, google::protobuf::MessageFactory* factory, const CelValue& value, Arena* arena) { const Message* msg = diff --git a/eval/public/structs/cel_proto_wrapper.h b/eval/public/structs/cel_proto_wrapper.h index 73942c253..8d334961c 100644 --- a/eval/public/structs/cel_proto_wrapper.h +++ b/eval/public/structs/cel_proto_wrapper.h @@ -1,9 +1,11 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAPPER_H_ #define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAPPER_H_ +#include + #include "google/protobuf/duration.pb.h" #include "google/protobuf/timestamp.pb.h" -#include "absl/types/optional.h" +#include "absl/base/nullability.h" #include "eval/public/cel_value.h" #include "internal/proto_time_encoding.h" #include "google/protobuf/arena.h" @@ -17,8 +19,19 @@ class CelProtoWrapper { // CreateMessage creates CelValue from google::protobuf::Message. // As some of CEL basic types are subclassing google::protobuf::Message, // this method contains type checking and downcasts. - static CelValue CreateMessage(const google::protobuf::Message* value, - google::protobuf::Arena* arena); + static CelValue CreateMessage(const google::protobuf::Message* absl_nonnull value, + const google::protobuf::DescriptorPool* absl_nonnull pool, + google::protobuf::MessageFactory* absl_nonnull factory, + google::protobuf::Arena* absl_nonnull arena); + + // Prefer using the overload that takes an explicit descriptor pool and + // message factory instead. This overload will use the ones associated with + // the value. + // + // For backward compatibility, nullptr message is allowed and will result in + // the CEL null_type value. + static CelValue CreateMessage(const google::protobuf::Message* absl_nullable value, + google::protobuf::Arena* absl_nullable arena); // Internal utility for creating a CelValue wrapping a user defined type. // Assumes that the message has been properly unpacked. @@ -43,7 +56,7 @@ class CelProtoWrapper { // message to native CelValue representation during a protobuf field read. // Just as CreateMessage should only be used when reading protobuf values, // MaybeWrapValue should only be used when assigning protobuf fields. - static absl::optional MaybeWrapValue( + static std::optional MaybeWrapValue( const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, const CelValue& value, google::protobuf::Arena* arena); }; diff --git a/eval/public/structs/cel_proto_wrapper_test.cc b/eval/public/structs/cel_proto_wrapper_test.cc index 408e33284..597943f7a 100644 --- a/eval/public/structs/cel_proto_wrapper_test.cc +++ b/eval/public/structs/cel_proto_wrapper_test.cc @@ -104,7 +104,7 @@ class CelProtoWrapperTest : public ::testing::Test { T dyn_value; CelValue cel_dyn_value = - CelProtoWrapper::CreateMessage(ReflectedCopy(message).get(), arena()); + CelProtoWrapper::CreateMessage(ReflectedCopy(message), arena()); EXPECT_THAT(cel_dyn_value.type(), Eq(cel_value.type())); EXPECT_TRUE(cel_dyn_value.GetValue(&dyn_value)); EXPECT_THAT(value, Eq(dyn_value)); @@ -121,10 +121,9 @@ class CelProtoWrapperTest : public ::testing::Test { EXPECT_THAT(cel_value.MessageOrDie(), testutil::EqualsProto(*result)); } - std::unique_ptr ReflectedCopy( - const google::protobuf::Message& message) { - std::unique_ptr dynamic_value( - factory_.GetPrototype(message.GetDescriptor())->New()); + google::protobuf::Message* ReflectedCopy(const google::protobuf::Message& message) { + google::protobuf::Message* dynamic_value = + factory_.GetPrototype(message.GetDescriptor())->New(&arena_); dynamic_value->CopyFrom(message); return dynamic_value; } @@ -213,7 +212,7 @@ TEST_F(CelProtoWrapperTest, UnwrapDynamicValueNull) { value_msg.set_null_value(protobuf::NULL_VALUE); CelValue value = - CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg).get(), arena()); + CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg), arena()); EXPECT_TRUE(value.IsNull()); } @@ -314,8 +313,8 @@ TEST_F(CelProtoWrapperTest, UnwrapDynamicStruct) { const std::string kFieldBool = "field_bool"; (*struct_msg.mutable_fields())[kFieldInt].set_number_value(1.); (*struct_msg.mutable_fields())[kFieldBool].set_bool_value(true); - CelValue value = - CelProtoWrapper::CreateMessage(ReflectedCopy(struct_msg).get(), arena()); + auto reflected_copy = ReflectedCopy(struct_msg); + CelValue value = CelProtoWrapper::CreateMessage(reflected_copy, arena()); EXPECT_TRUE(value.IsMap()); const CelMap* cel_map = value.MapOrDie(); ASSERT_TRUE(cel_map != nullptr); @@ -355,7 +354,7 @@ TEST_F(CelProtoWrapperTest, UnwrapDynamicValueStruct) { .set_number_value(2); CelValue value = - CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg).get(), arena()); + CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg), arena()); EXPECT_TRUE(value.IsMap()); EXPECT_TRUE( (*value.MapOrDie())[CelValue::CreateString(&kField1)].has_value()); @@ -398,7 +397,7 @@ TEST_F(CelProtoWrapperTest, UnwrapDynamicValueListValue) { value_msg.mutable_list_value()->add_values()->set_number_value(2.); CelValue value = - CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg).get(), arena()); + CelProtoWrapper::CreateMessage(ReflectedCopy(value_msg), arena()); EXPECT_TRUE(value.IsList()); EXPECT_THAT((*value.ListOrDie())[0].DoubleOrDie(), testing::DoubleEq(1)); EXPECT_THAT((*value.ListOrDie())[1].DoubleOrDie(), testing::DoubleEq(2)); @@ -426,6 +425,47 @@ TEST_F(CelProtoWrapperTest, UnwrapInvalidAny) { ASSERT_TRUE(CelProtoWrapper::CreateMessage(&any, arena()).IsError()); } +TEST_F(CelProtoWrapperTest, CreateMessageExplicitPoolAndFactory) { + TestMessage test_message; + test_message.set_string_value("test"); + + CelValue value = CelProtoWrapper::CreateMessage( + &test_message, google::protobuf::DescriptorPool::generated_pool(), + google::protobuf::MessageFactory::generated_factory(), arena()); + ASSERT_TRUE(value.IsMessage()); + EXPECT_THAT(value.MessageOrDie(), testutil::EqualsProto(test_message)); +} + +TEST_F(CelProtoWrapperTest, CreateMessageExplicitPoolAndFactoryUnpackAny) { + TestMessage test_message; + test_message.set_string_value("test"); + + Any any; + any.PackFrom(test_message); + + google::protobuf::DynamicMessageFactory factory( + google::protobuf::DescriptorPool::generated_pool()); + CelValue value = CelProtoWrapper::CreateMessage( + &any, google::protobuf::DescriptorPool::generated_pool(), &factory, arena()); + ASSERT_TRUE(value.IsMessage()); + EXPECT_THAT(value.MessageOrDie(), testutil::EqualsProto(test_message)); +} + +TEST_F(CelProtoWrapperTest, + CreateMessageExplicitPoolAndFactoryUnpackAnyNotFound) { + TestMessage test_message; + test_message.set_string_value("test"); + + Any any; + any.PackFrom(test_message); + + google::protobuf::DescriptorPool empty_pool; + google::protobuf::DynamicMessageFactory factory(&empty_pool); + CelValue value = + CelProtoWrapper::CreateMessage(&any, &empty_pool, &factory, arena()); + EXPECT_TRUE(value.IsError()); +} + // Test support of google.protobuf.Value wrappers in CelValue. TEST_F(CelProtoWrapperTest, UnwrapBoolWrapper) { bool value = true; From c221fdd235fb3370678d1a2589819f16bbbd7b0e Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 9 Sep 2026 11:04:08 -0700 Subject: [PATCH 210/350] Split out utility for wrapping values into WKTs. PiperOrigin-RevId: 978642463 --- common/BUILD | 2 +- common/legacy_value.cc | 2 +- eval/public/structs/BUILD | 68 +- eval/public/structs/cel_proto_wrap_util.cc | 826 ---------------- eval/public/structs/cel_proto_wrap_util.h | 12 - .../structs/cel_proto_wrap_util_test.cc | 528 ----------- .../cel_proto_wrap_value_to_message.cc | 895 ++++++++++++++++++ .../structs/cel_proto_wrap_value_to_message.h | 40 + .../cel_proto_wrap_value_to_message_test.cc | 616 ++++++++++++ eval/public/structs/cel_proto_wrapper.cc | 2 +- eval/public/structs/field_access_impl.cc | 1 + 11 files changed, 1614 insertions(+), 1378 deletions(-) create mode 100644 eval/public/structs/cel_proto_wrap_value_to_message.cc create mode 100644 eval/public/structs/cel_proto_wrap_value_to_message.h create mode 100644 eval/public/structs/cel_proto_wrap_value_to_message_test.cc diff --git a/common/BUILD b/common/BUILD index 185af4e27..f34f07fcf 100644 --- a/common/BUILD +++ b/common/BUILD @@ -793,7 +793,7 @@ cc_library( "//eval/internal:cel_value_equal", "//eval/public:cel_value", "//eval/public:message_wrapper", - "//eval/public/structs:cel_proto_wrap_util", + "//eval/public/structs:cel_proto_wrap_value_to_message", "//eval/public/structs:legacy_type_info_apis", "//eval/public/structs:proto_message_type_adapter", "//eval/public/structs:trivial_legacy_type_info_internal", diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 34b0e000f..3a2108c4d 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -51,7 +51,7 @@ #include "eval/internal/cel_value_equal.h" #include "eval/public/cel_value.h" #include "eval/public/message_wrapper.h" -#include "eval/public/structs/cel_proto_wrap_util.h" +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" #include "eval/public/structs/legacy_type_info_apis.h" #include "eval/public/structs/proto_message_type_adapter.h" #include "eval/public/structs/trivial_legacy_type_info_internal.h" diff --git a/eval/public/structs/BUILD b/eval/public/structs/BUILD index 41c0d7047..107451e01 100644 --- a/eval/public/structs/BUILD +++ b/eval/public/structs/BUILD @@ -28,7 +28,7 @@ cc_library( "cel_proto_wrapper.h", ], deps = [ - ":cel_proto_wrap_util", + ":cel_proto_wrap_value_to_message", ":proto_message_type_adapter", ":trivial_legacy_type_info_internal", "//common:value", @@ -69,16 +69,13 @@ cc_library( deps = [ ":protobuf_value_factory", "//eval/public:cel_value", - "//internal:overflow", "//internal:proto_time_encoding", "//internal:status_macros", - "//internal:time", "//internal:well_known_types", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:overload", "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -104,19 +101,70 @@ cc_test( ], deps = [ ":cel_proto_wrap_util", - ":protobuf_value_factory", ":trivial_legacy_type_info", "//eval/public:cel_value", - "//eval/public:message_wrapper", - "//eval/public/containers:container_backed_list_impl", - "//eval/public/containers:container_backed_map_impl", "//eval/testutil:test_message_cc_proto", "//internal:proto_time_encoding", + "//internal:testing", + "//testutil:util", + "@com_google_absl//absl/status", + "@com_google_protobuf//:any_cc_proto", + "@com_google_protobuf//:duration_cc_proto", + "@com_google_protobuf//:empty_cc_proto", + "@com_google_protobuf//:field_mask_cc_proto", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:struct_cc_proto", + "@com_google_protobuf//:wrappers_cc_proto", + ], +) + +cc_library( + name = "cel_proto_wrap_value_to_message", + srcs = [ + "cel_proto_wrap_value_to_message.cc", + ], + hdrs = [ + "cel_proto_wrap_value_to_message.h", + ], + deps = [ + "//eval/public:cel_value", + "//internal:overflow", + "//internal:proto_time_encoding", "//internal:status_macros", + "//internal:time", + "//internal:well_known_types", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", + "@com_google_absl//absl/time", + "@com_google_protobuf//:any_cc_proto", + "@com_google_protobuf//:duration_cc_proto", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:struct_cc_proto", + "@com_google_protobuf//:timestamp_cc_proto", + "@com_google_protobuf//:wrappers_cc_proto", + ], +) + +cc_test( + name = "cel_proto_wrap_value_to_message_test", + size = "small", + srcs = [ + "cel_proto_wrap_value_to_message_test.cc", + ], + deps = [ + ":cel_proto_wrap_value_to_message", + ":trivial_legacy_type_info", + "//eval/public:cel_value", + "//eval/public/containers:container_backed_list_impl", + "//eval/public/containers:container_backed_map_impl", + "//eval/testutil:test_message_cc_proto", "//internal:testing", "//testutil:util", - "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", "@com_google_absl//absl/types:span", @@ -126,6 +174,7 @@ cc_test( "@com_google_protobuf//:field_mask_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:struct_cc_proto", + "@com_google_protobuf//:timestamp_cc_proto", "@com_google_protobuf//:wrappers_cc_proto", ], ) @@ -140,6 +189,7 @@ cc_library( ], deps = [ ":cel_proto_wrap_util", + ":cel_proto_wrap_value_to_message", ":protobuf_value_factory", "//eval/public:cel_options", "//eval/public:cel_value", diff --git a/eval/public/structs/cel_proto_wrap_util.cc b/eval/public/structs/cel_proto_wrap_util.cc index 490686fcb..87d05ee3f 100644 --- a/eval/public/structs/cel_proto_wrap_util.cc +++ b/eval/public/structs/cel_proto_wrap_util.cc @@ -16,7 +16,6 @@ #include #include -#include #include #include #include @@ -32,11 +31,9 @@ #include "absl/base/optimization.h" #include "absl/functional/overload.h" #include "absl/log/absl_check.h" -#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" -#include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" @@ -45,16 +42,12 @@ #include "absl/types/variant.h" #include "eval/public/cel_value.h" #include "eval/public/structs/protobuf_value_factory.h" -#include "internal/overflow.h" #include "internal/proto_time_encoding.h" #include "internal/status_macros.h" -#include "internal/time.h" #include "internal/well_known_types.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" -#include "google/protobuf/json/json.h" #include "google/protobuf/message.h" -#include "google/protobuf/message_lite.h" namespace google::api::expr::runtime::internal { @@ -82,55 +75,6 @@ using google::protobuf::Descriptor; using google::protobuf::DescriptorPool; using google::protobuf::Message; using google::protobuf::MessageFactory; -using google::protobuf::json::MessageToJsonString; -using google::protobuf::json::PrintOptions; - -// kMaxIntJSON is defined as the Number.MAX_SAFE_INTEGER value per EcmaScript 6. -constexpr int64_t kMaxIntJSON = (1ll << 53) - 1; - -// kMinIntJSON is defined as the Number.MIN_SAFE_INTEGER value per EcmaScript 6. -constexpr int64_t kMinIntJSON = -kMaxIntJSON; - -// IsJSONSafe indicates whether the int is safely representable as a floating -// point value in JSON. -static bool IsJSONSafe(int64_t i) { - return i >= kMinIntJSON && i <= kMaxIntJSON; -} - -// IsJSONSafe indicates whether the uint is safely representable as a floating -// point value in JSON. -static bool IsJSONSafe(uint64_t i) { - return i <= static_cast(kMaxIntJSON); -} - -static bool IsEmptyProto(const google::protobuf::Descriptor* descriptor) { - return descriptor->full_name() == "google.protobuf.Empty"; -} - -static bool IsFieldMaskProto(const google::protobuf::Descriptor* descriptor) { - return descriptor->full_name() == "google.protobuf.FieldMask"; -} - -static std::optional GetFieldMaskJsonString( - const google::protobuf::Message& message) { - // TODO(b/540507668): Refactor to pipe descriptor_pool through - // ValueFromValue to use internal::MessageToJson. - PrintOptions json_options; - std::string json_str; - auto status = MessageToJsonString(message, &json_str, json_options); - if (!status.ok()) { - ABSL_LOG(ERROR) << "Failed to convert FieldMask to JSON: " << status; - return std::nullopt; - } - // If JSON marshalling is correct, we know we'll always get a plain - // JSON string value and it shouldn't contain any escapes that we need - // to interpret. - if (json_str.size() >= 2 && json_str.front() == '"' && - json_str.back() == '"') { - return json_str.substr(1, json_str.size() - 2); - } - return json_str; -} // Map implementation wrapping google.protobuf.ListValue class DynamicList : public CelList { @@ -756,768 +700,6 @@ absl::optional DynamicMap::operator[](CelValue key) const { return ValueManager(factory_, arena_).ValueFromMessage(&it->second); } -google::protobuf::Message* DurationFromValue(const google::protobuf::Message* prototype, - const CelValue& value, - google::protobuf::Arena* arena) { - absl::Duration val; - if (!value.GetValue(&val)) { - return nullptr; - } - if (!cel::internal::ValidateDuration(val).ok()) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetDurationReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.UnsafeSetFromAbslDuration(message, val); - return message; -} - -google::protobuf::Message* BoolFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - bool val; - if (!value.GetValue(&val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetBoolValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, val); - return message; -} - -google::protobuf::Message* BytesFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - CelValue::BytesHolder view_val; - if (!value.GetValue(&view_val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetBytesValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, view_val.value()); - return message; -} - -google::protobuf::Message* DoubleFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - double val; - if (!value.GetValue(&val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetDoubleValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, val); - return message; -} - -google::protobuf::Message* FloatFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - double val; - if (!value.GetValue(&val)) { - return nullptr; - } - float fval = val; - // Abort the conversion if the value is outside the float range. - if (val > std::numeric_limits::max()) { - fval = std::numeric_limits::infinity(); - } else if (val < std::numeric_limits::lowest()) { - fval = -std::numeric_limits::infinity(); - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetFloatValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, static_cast(fval)); - return message; -} - -google::protobuf::Message* Int32FromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - int64_t val; - if (!value.GetValue(&val)) { - return nullptr; - } - if (!cel::internal::CheckedInt64ToInt32(val).ok()) { - return nullptr; - } - int32_t ival = static_cast(val); - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetInt32ValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, ival); - return message; -} - -google::protobuf::Message* Int64FromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - int64_t val; - if (!value.GetValue(&val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetInt64ValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, val); - return message; -} - -google::protobuf::Message* StringFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - CelValue::StringHolder view_val; - if (!value.GetValue(&view_val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetStringValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, view_val.value()); - return message; -} - -google::protobuf::Message* TimestampFromValue(const google::protobuf::Message* prototype, - const CelValue& value, - google::protobuf::Arena* arena) { - absl::Time val; - if (!value.GetValue(&val)) { - return nullptr; - } - if (!cel::internal::ValidateTimestamp(val).ok()) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetTimestampReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.UnsafeSetFromAbslTime(message, val); - return message; -} - -google::protobuf::Message* UInt32FromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - uint64_t val; - if (!value.GetValue(&val)) { - return nullptr; - } - if (!cel::internal::CheckedUint64ToUint32(val).ok()) { - return nullptr; - } - uint32_t ival = static_cast(val); - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetUInt32ValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, ival); - return message; -} - -google::protobuf::Message* UInt64FromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - uint64_t val; - if (!value.GetValue(&val)) { - return nullptr; - } - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetUInt64ValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetValue(message, val); - return message; -} - -google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, const CelValue& value, - google::protobuf::Arena* arena); - -google::protobuf::Message* ValueFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - return ValueFromValue(prototype->New(arena), value, arena); -} - -google::protobuf::Message* ListFromValue(google::protobuf::Message* message, const CelValue& value, - google::protobuf::Arena* arena) { - if (!value.IsList()) { - return nullptr; - } - const CelList& list = *value.ListOrDie(); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetListValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - for (int i = 0; i < list.size(); i++) { - auto e = list.Get(arena, i); - auto* elem = reflection.AddValues(message); - if (ValueFromValue(elem, e, arena) == nullptr) { - return nullptr; - } - } - return message; -} - -google::protobuf::Message* ListFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - if (!value.IsList()) { - return nullptr; - } - return ListFromValue(prototype->New(arena), value, arena); -} - -google::protobuf::Message* StructFromValue(google::protobuf::Message* message, - const CelValue& value, google::protobuf::Arena* arena) { - if (!value.IsMap()) { - return nullptr; - } - const CelMap& map = *value.MapOrDie(); - absl::StatusOr keys_or = map.ListKeys(arena); - if (!keys_or.ok()) { - // If map doesn't support listing keys, it can't pack into a Struct value. - // This will surface as a CEL error when the object creation expression - // fails. - return nullptr; - } - const CelList& keys = **keys_or; - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetStructReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - for (int i = 0; i < keys.size(); i++) { - auto k = keys.Get(arena, i); - // If the key is not a string type, abort the conversion. - if (!k.IsString()) { - return nullptr; - } - absl::string_view key = k.StringOrDie().value(); - - auto v = map.Get(arena, k); - if (!v.has_value()) { - return nullptr; - } - auto* field = reflection.InsertField(message, key); - if (ValueFromValue(field, *v, arena) == nullptr) { - return nullptr; - } - } - return message; -} - -google::protobuf::Message* StructFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - if (!value.IsMap()) { - return nullptr; - } - return StructFromValue(prototype->New(arena), value, arena); -} - -google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, const CelValue& value, - google::protobuf::Arena* arena) { - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetValueReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - switch (value.type()) { - case CelValue::Type::kBool: { - bool val; - if (value.GetValue(&val)) { - reflection.SetBoolValue(message, val); - return message; - } - } break; - case CelValue::Type::kBytes: { - // Base64 encode byte strings to ensure they can safely be transported - // in a JSON string. - CelValue::BytesHolder val; - if (value.GetValue(&val)) { - reflection.SetStringValueFromBytes(message, val.value()); - return message; - } - } break; - case CelValue::Type::kDouble: { - double val; - if (value.GetValue(&val)) { - reflection.SetNumberValue(message, val); - return message; - } - } break; - case CelValue::Type::kDuration: { - // Convert duration values to a protobuf JSON format. - absl::Duration val; - if (value.GetValue(&val)) { - CEL_RETURN_IF_ERROR(cel::internal::ValidateDuration(val)) - .With(IgnoreErrorAndReturnNullptr()); - reflection.SetStringValueFromDuration(message, val); - return message; - } - } break; - case CelValue::Type::kInt64: { - int64_t val; - // Convert int64_t values within the int53 range to doubles, otherwise - // serialize the value to a string. - if (value.GetValue(&val)) { - reflection.SetNumberValue(message, val); - return message; - } - } break; - case CelValue::Type::kString: { - CelValue::StringHolder val; - if (value.GetValue(&val)) { - reflection.SetStringValue(message, val.value()); - return message; - } - } break; - case CelValue::Type::kTimestamp: { - // Convert timestamp values to a protobuf JSON format. - absl::Time val; - if (value.GetValue(&val)) { - CEL_RETURN_IF_ERROR(cel::internal::ValidateTimestamp(val)) - .With(IgnoreErrorAndReturnNullptr()); - reflection.SetStringValueFromTimestamp(message, val); - return message; - } - } break; - case CelValue::Type::kUint64: { - uint64_t val; - // Convert uint64_t values within the int53 range to doubles, otherwise - // serialize the value to a string. - if (value.GetValue(&val)) { - reflection.SetNumberValue(message, val); - return message; - } - } break; - case CelValue::Type::kList: { - if (ListFromValue(reflection.MutableListValue(message), value, arena) != - nullptr) { - return message; - } - } break; - case CelValue::Type::kMap: { - if (StructFromValue(reflection.MutableStructValue(message), value, - arena) != nullptr) { - return message; - } - } break; - case CelValue::Type::kMessage: { - const google::protobuf::Message* message_ptr = value.MessageOrDie(); - if (IsEmptyProto(message_ptr->GetDescriptor())) { - reflection.MutableStructValue(message); - return message; - } - if (IsFieldMaskProto(message_ptr->GetDescriptor())) { - std::optional fm_str = - GetFieldMaskJsonString(*message_ptr); - if (fm_str.has_value()) { - reflection.SetStringValue(message, *fm_str); - return message; - } - return nullptr; - } - return nullptr; - } break; - case CelValue::Type::kNullType: - reflection.SetNullValue(message); - return message; - break; - default: - return nullptr; - } - return nullptr; -} - -bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* arena); - -bool ListFromValue(ListValue* json_list, const CelValue& value, - google::protobuf::Arena* arena) { - if (!value.IsList()) { - return false; - } - const CelList& list = *value.ListOrDie(); - for (int i = 0; i < list.size(); i++) { - auto e = list.Get(arena, i); - Value* elem = json_list->add_values(); - if (!ValueFromValue(elem, e, arena)) { - return false; - } - } - return true; -} - -bool StructFromValue(Struct* json_struct, const CelValue& value, - google::protobuf::Arena* arena) { - if (!value.IsMap()) { - return false; - } - const CelMap& map = *value.MapOrDie(); - absl::StatusOr keys_or = map.ListKeys(arena); - if (!keys_or.ok()) { - // If map doesn't support listing keys, it can't pack into a Struct value. - // This will surface as a CEL error when the object creation expression - // fails. - return false; - } - const CelList& keys = **keys_or; - auto fields = json_struct->mutable_fields(); - for (int i = 0; i < keys.size(); i++) { - auto k = keys.Get(arena, i); - // If the key is not a string type, abort the conversion. - if (!k.IsString()) { - return false; - } - absl::string_view key = k.StringOrDie().value(); - - auto v = map.Get(arena, k); - if (!v.has_value()) { - return false; - } - Value field_value; - if (!ValueFromValue(&field_value, *v, arena)) { - return false; - } - (*fields)[std::string(key)] = field_value; - } - return true; -} - -bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* arena) { - switch (value.type()) { - case CelValue::Type::kBool: { - bool val; - if (value.GetValue(&val)) { - json->set_bool_value(val); - return true; - } - } break; - case CelValue::Type::kBytes: { - // Base64 encode byte strings to ensure they can safely be transported - // in a JSON string. - CelValue::BytesHolder val; - if (value.GetValue(&val)) { - json->set_string_value(absl::Base64Escape(val.value())); - return true; - } - } break; - case CelValue::Type::kDouble: { - double val; - if (value.GetValue(&val)) { - json->set_number_value(val); - return true; - } - } break; - case CelValue::Type::kDuration: { - // Convert duration values to a protobuf JSON format. - absl::Duration val; - if (value.GetValue(&val)) { - auto encode = cel::internal::EncodeDurationToString(val); - if (!encode.ok()) { - return false; - } - json->set_string_value(*encode); - return true; - } - } break; - case CelValue::Type::kInt64: { - int64_t val; - // Convert int64_t values within the int53 range to doubles, otherwise - // serialize the value to a string. - if (value.GetValue(&val)) { - if (IsJSONSafe(val)) { - json->set_number_value(val); - } else { - json->set_string_value(absl::StrCat(val)); - } - return true; - } - } break; - case CelValue::Type::kString: { - CelValue::StringHolder val; - if (value.GetValue(&val)) { - json->set_string_value(val.value()); - return true; - } - } break; - case CelValue::Type::kTimestamp: { - // Convert timestamp values to a protobuf JSON format. - absl::Time val; - if (value.GetValue(&val)) { - auto encode = cel::internal::EncodeTimeToString(val); - if (!encode.ok()) { - return false; - } - json->set_string_value(*encode); - return true; - } - } break; - case CelValue::Type::kUint64: { - uint64_t val; - // Convert uint64_t values within the int53 range to doubles, otherwise - // serialize the value to a string. - if (value.GetValue(&val)) { - if (IsJSONSafe(val)) { - json->set_number_value(val); - } else { - json->set_string_value(absl::StrCat(val)); - } - return true; - } - } break; - case CelValue::Type::kList: - return ListFromValue(json->mutable_list_value(), value, arena); - case CelValue::Type::kMap: - return StructFromValue(json->mutable_struct_value(), value, arena); - case CelValue::Type::kMessage: { - const google::protobuf::Message* message_ptr = value.MessageOrDie(); - if (IsEmptyProto(message_ptr->GetDescriptor())) { - json->mutable_struct_value(); - return true; - } - if (IsFieldMaskProto(message_ptr->GetDescriptor())) { - std::optional fm_str = - GetFieldMaskJsonString(*message_ptr); - if (fm_str.has_value()) { - json->set_string_value(*fm_str); - return true; - } - return false; - } - return false; - } - case CelValue::Type::kNullType: - json->set_null_value(protobuf::NULL_VALUE); - return true; - default: - return false; - } - return false; -} - -google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototype, - const CelValue& value, google::protobuf::Arena* arena) { - std::string type_name; - absl::Cord payload; - - // In open source, any->PackFrom() returns void rather than boolean. - switch (value.type()) { - case CelValue::Type::kBool: { - BoolValue v; - type_name = v.GetTypeName(); - v.set_value(value.BoolOrDie()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kBytes: { - BytesValue v; - type_name = v.GetTypeName(); - v.set_value(value.BytesOrDie().value()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kDouble: { - DoubleValue v; - type_name = v.GetTypeName(); - v.set_value(value.DoubleOrDie()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kDuration: { - Duration v; - if (!cel::internal::EncodeDuration(value.DurationOrDie(), &v).ok()) { - return nullptr; - } - type_name = v.GetTypeName(); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kInt64: { - Int64Value v; - type_name = v.GetTypeName(); - v.set_value(value.Int64OrDie()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kString: { - StringValue v; - type_name = v.GetTypeName(); - v.set_value(std::string(value.StringOrDie().value())); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kTimestamp: { - Timestamp v; - if (!cel::internal::EncodeTime(value.TimestampOrDie(), &v).ok()) { - return nullptr; - } - type_name = v.GetTypeName(); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kUint64: { - UInt64Value v; - type_name = v.GetTypeName(); - v.set_value(value.Uint64OrDie()); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kList: { - ListValue v; - if (!ListFromValue(&v, value, arena)) { - return nullptr; - } - type_name = v.GetTypeName(); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kMap: { - Struct v; - if (!StructFromValue(&v, value, arena)) { - return nullptr; - } - type_name = v.GetTypeName(); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kNullType: { - Value v; - type_name = v.GetTypeName(); - v.set_null_value(google::protobuf::NULL_VALUE); - payload = v.SerializeAsCord(); - } break; - case CelValue::Type::kMessage: { - type_name = value.MessageWrapperOrDie().message_ptr()->GetTypeName(); - payload = value.MessageWrapperOrDie().message_ptr()->SerializeAsCord(); - } break; - default: - return nullptr; - } - - auto* message = prototype->New(arena); - CEL_ASSIGN_OR_RETURN( - auto reflection, - cel::well_known_types::GetAnyReflection(message->GetDescriptor()), - _.With(IgnoreErrorAndReturnNullptr())); - reflection.SetTypeUrl(message, - absl::StrCat("type.googleapis.com/", type_name)); - reflection.SetValue(message, payload); - return message; -} - -bool IsAlreadyWrapped(google::protobuf::Descriptor::WellKnownType wkt, - const CelValue& value) { - if (value.IsMessage()) { - const auto* msg = value.MessageOrDie(); - if (wkt == msg->GetDescriptor()->well_known_type()) { - return true; - } - } - return false; -} - -// MessageFromValueMaker makes a specific protobuf Message instance based on -// the desired protobuf type name and an input CelValue. -// -// It holds a registry of CelValue factories for specific subtypes of Message. -// If message does not match any of types stored in registry, an the factory -// returns an absent value. -class MessageFromValueMaker { - public: - // Non-copyable, non-assignable - MessageFromValueMaker(const MessageFromValueMaker&) = delete; - MessageFromValueMaker& operator=(const MessageFromValueMaker&) = delete; - - static google::protobuf::Message* MaybeWrapMessage(const google::protobuf::Descriptor* descriptor, - google::protobuf::MessageFactory* factory, - const CelValue& value, - Arena* arena) { - switch (descriptor->well_known_type()) { - case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return DoubleFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return FloatFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return Int64FromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return UInt64FromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return Int32FromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return UInt32FromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return StringFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return BytesFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_BOOLVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return BoolFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_ANY: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return AnyFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return DurationFromValue(factory->GetPrototype(descriptor), value, - arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return TimestampFromValue(factory->GetPrototype(descriptor), value, - arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return ValueFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return ListFromValue(factory->GetPrototype(descriptor), value, arena); - case google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT: - if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { - return nullptr; - } - return StructFromValue(factory->GetPrototype(descriptor), value, arena); - // WELLKNOWNTYPE_FIELDMASK has no special CelValue type - default: - return nullptr; - } - } -}; - } // namespace CelValue UnwrapMessageToValue(const google::protobuf::Message* value, @@ -1536,12 +718,4 @@ CelValue UnwrapMessageToValue(const google::protobuf::Message* value, return factory(value); } -const google::protobuf::Message* MaybeWrapValueToMessage( - const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, - const CelValue& value, Arena* arena) { - google::protobuf::Message* msg = MessageFromValueMaker::MaybeWrapMessage( - descriptor, factory, value, arena); - return msg; -} - } // namespace google::api::expr::runtime::internal diff --git a/eval/public/structs/cel_proto_wrap_util.h b/eval/public/structs/cel_proto_wrap_util.h index 508985209..1d15c9ead 100644 --- a/eval/public/structs/cel_proto_wrap_util.h +++ b/eval/public/structs/cel_proto_wrap_util.h @@ -28,18 +28,6 @@ CelValue UnwrapMessageToValue(const google::protobuf::Message* value, const ProtobufValueFactory& factory, google::protobuf::Arena* arena); -// MaybeWrapValue attempts to wrap the input value in a proto message with -// the given type_name. If the value can be wrapped, it is returned as a -// protobuf message. Otherwise, the result will be nullptr. -// -// This method is the complement to MaybeUnwrapValue which may unwrap a protobuf -// message to native CelValue representation during a protobuf field read. -// Just as CreateMessage should only be used when reading protobuf values, -// MaybeWrapValue should only be used when assigning protobuf fields. -const google::protobuf::Message* MaybeWrapValueToMessage( - const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, - const CelValue& value, google::protobuf::Arena* arena); - } // namespace google::api::expr::runtime::internal #endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAP_UTIL_H_ diff --git a/eval/public/structs/cel_proto_wrap_util_test.cc b/eval/public/structs/cel_proto_wrap_util_test.cc index b196d1dd3..bbcb64060 100644 --- a/eval/public/structs/cel_proto_wrap_util_test.cc +++ b/eval/public/structs/cel_proto_wrap_util_test.cc @@ -16,10 +16,8 @@ #include #include -#include #include #include -#include #include #include "google/protobuf/any.pb.h" @@ -28,20 +26,11 @@ #include "google/protobuf/field_mask.pb.h" #include "google/protobuf/struct.pb.h" #include "google/protobuf/wrappers.pb.h" -#include "absl/base/no_destructor.h" #include "absl/status/status.h" -#include "absl/strings/str_cat.h" -#include "absl/time/time.h" -#include "absl/types/span.h" #include "eval/public/cel_value.h" -#include "eval/public/containers/container_backed_list_impl.h" -#include "eval/public/containers/container_backed_map_impl.h" -#include "eval/public/message_wrapper.h" -#include "eval/public/structs/protobuf_value_factory.h" #include "eval/public/structs/trivial_legacy_type_info.h" #include "eval/testutil/test_message.pb.h" #include "internal/proto_time_encoding.h" -#include "internal/status_macros.h" #include "internal/testing.h" #include "testutil/util.h" #include "google/protobuf/arena.h" @@ -52,12 +41,10 @@ namespace google::api::expr::runtime::internal { namespace { -using ::google::protobuf::TextFormat; using ::testing::Eq; using ::testing::UnorderedPointwise; using google::protobuf::Duration; -using google::protobuf::FieldMask; using google::protobuf::ListValue; using google::protobuf::Struct; using google::protobuf::Timestamp; @@ -85,38 +72,7 @@ class CelProtoWrapperTest : public ::testing::Test { protected: CelProtoWrapperTest() = default; - void ExpectWrappedMessage(const CelValue& value, - const google::protobuf::Message& message) { - // Test the input value wraps to the destination message type. - auto* result = MaybeWrapValueToMessage( - message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), - value, arena()); - EXPECT_TRUE(result != nullptr); - EXPECT_THAT(result, testutil::EqualsProto(message)); - - // Ensure that double wrapping results in the object being wrapped once. - auto* identity = MaybeWrapValueToMessage( - message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), - ProtobufValueFactoryImpl(result), arena()); - EXPECT_TRUE(identity == nullptr); - - // Check to make sure that even dynamic messages can be used as input to - // the wrapping call. - result = MaybeWrapValueToMessage( - ReflectedCopy(message)->GetDescriptor(), - ReflectedCopy(message)->GetReflection()->GetMessageFactory(), value, - arena()); - EXPECT_TRUE(result != nullptr); - EXPECT_THAT(result, testutil::EqualsProto(message)); - } - void ExpectNotWrapped(const CelValue& value, const google::protobuf::Message& message) { - // Test the input value does not wrap by asserting value == result. - auto result = MaybeWrapValueToMessage( - message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), - value, arena()); - EXPECT_TRUE(result == nullptr); - } template void ExpectUnwrappedPrimitive(const google::protobuf::Message& message, T result) { @@ -442,120 +398,6 @@ TEST_F(CelProtoWrapperTest, UnwrapInvalidAny) { UnwrapMessageToValue(&any, &ProtobufValueFactoryImpl, arena()).IsError()); } -TEST_F(CelProtoWrapperTest, WrapFieldMaskToValue) { - FieldMask field_mask; - ASSERT_TRUE(TextFormat::ParseFromString(R"pb( - paths: "foo.bar" paths: "baz" - )pb", - &field_mask)); - CelValue value = ProtobufValueFactoryImpl(&field_mask); - - Value expected_message; - ASSERT_TRUE(TextFormat::ParseFromString(R"pb(string_value: "foo.bar,baz")pb", - &expected_message)); - - ExpectWrappedMessage(value, expected_message); -} - -TEST_F(CelProtoWrapperTest, WrapMapWithFieldMaskToAny) { - const std::string kField = "field_mask"; - FieldMask field_mask; - ASSERT_TRUE(TextFormat::ParseFromString(R"pb( - paths: "foo.bar" paths: "baz" - )pb", - &field_mask)); - CelValue value = ProtobufValueFactoryImpl(&field_mask); - - std::vector> args = { - {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr cel_map, - CreateContainerBackedMap( - absl::Span>(args.data(), args.size()))); - CelValue cel_value = CelValue::CreateMap(cel_map.get()); - - Struct expected_struct; - ASSERT_TRUE( - TextFormat::ParseFromString(R"pb( - fields { - key: "field_mask" - value { string_value: "foo.bar,baz" } - } - )pb", - &expected_struct)); - Any expected_message; - ASSERT_TRUE(expected_message.PackFrom(expected_struct)); - - ExpectWrappedMessage(cel_value, expected_message); -} - -TEST_F(CelProtoWrapperTest, WrapListWithFieldMaskToAny) { - FieldMask field_mask; - ASSERT_TRUE(TextFormat::ParseFromString(R"pb( - paths: "foo.bar" paths: "baz" - )pb", - &field_mask)); - CelValue value = ProtobufValueFactoryImpl(&field_mask); - - std::vector list_entries = {value}; - ContainerBackedListImpl cel_list(list_entries); - CelValue list_value = CelValue::CreateList(&cel_list); - - ListValue expected_list; - ASSERT_TRUE(TextFormat::ParseFromString( - R"pb(values { string_value: "foo.bar,baz" })pb", &expected_list)); - Any expected_message; - ASSERT_TRUE(expected_message.PackFrom(expected_list)); - - ExpectWrappedMessage(list_value, expected_message); -} - -TEST_F(CelProtoWrapperTest, WrapEmptyToValue) { - google::protobuf::Empty empty; - CelValue value = ProtobufValueFactoryImpl(&empty); - - Value expected_message; - expected_message.mutable_struct_value(); - - ExpectWrappedMessage(value, expected_message); -} - -TEST_F(CelProtoWrapperTest, WrapMapWithEmptyToAny) { - const std::string kField = "empty"; - google::protobuf::Empty empty; - CelValue value = ProtobufValueFactoryImpl(&empty); - - std::vector> args = { - {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr cel_map, - CreateContainerBackedMap( - absl::Span>(args.data(), args.size()))); - auto cel_value = CelValue::CreateMap(cel_map.get()); - - Struct expected_struct; - (*expected_struct.mutable_fields())[kField].mutable_struct_value(); - Any expected_message; - expected_message.PackFrom(expected_struct); - - ExpectWrappedMessage(cel_value, expected_message); -} - -TEST_F(CelProtoWrapperTest, WrapListWithEmptyToAny) { - google::protobuf::Empty empty; - CelValue value = ProtobufValueFactoryImpl(&empty); - - std::vector list_entries = {value}; - ContainerBackedListImpl cel_list(list_entries); - CelValue list_value = CelValue::CreateList(&cel_list); - - ListValue expected_list; - expected_list.add_values()->mutable_struct_value(); - Any expected_message; - expected_message.PackFrom(expected_list); - - ExpectWrappedMessage(list_value, expected_message); -} // Test support of google.protobuf.Value wrappers in CelValue. TEST_F(CelProtoWrapperTest, UnwrapBoolWrapper) { bool value = true; @@ -631,376 +473,6 @@ TEST_F(CelProtoWrapperTest, UnwrapBytesWrapper) { ExpectUnwrappedPrimitive(wrapper, value); } -TEST_F(CelProtoWrapperTest, WrapNull) { - auto cel_value = CelValue::CreateNull(); - - Value json; - json.set_null_value(protobuf::NULL_VALUE); - ExpectWrappedMessage(cel_value, json); - - Any any; - any.PackFrom(json); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapBool) { - auto cel_value = CelValue::CreateBool(true); - - Value json; - json.set_bool_value(true); - ExpectWrappedMessage(cel_value, json); - - BoolValue wrapper; - wrapper.set_value(true); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapBytes) { - std::string str = "hello world"; - auto cel_value = CelValue::CreateBytes(CelValue::BytesHolder(&str)); - - BytesValue wrapper; - wrapper.set_value(str); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapBytesToValue) { - std::string str = "hello world"; - auto cel_value = CelValue::CreateBytes(CelValue::BytesHolder(&str)); - - Value json; - json.set_string_value("aGVsbG8gd29ybGQ="); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapDuration) { - auto cel_value = CelValue::CreateDuration(absl::Seconds(300)); - - Duration d; - d.set_seconds(300); - ExpectWrappedMessage(cel_value, d); - - Any any; - any.PackFrom(d); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapDurationToValue) { - auto cel_value = CelValue::CreateDuration(absl::Seconds(300)); - - Value json; - json.set_string_value("300s"); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapDouble) { - double num = 1.5; - auto cel_value = CelValue::CreateDouble(num); - - Value json; - json.set_number_value(num); - ExpectWrappedMessage(cel_value, json); - - DoubleValue wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapDoubleToFloatValue) { - double num = 1.5; - auto cel_value = CelValue::CreateDouble(num); - - FloatValue wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); - - // Imprecise double -> float representation results in truncation. - double small_num = -9.9e-100; - wrapper.set_value(small_num); - cel_value = CelValue::CreateDouble(small_num); - ExpectWrappedMessage(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapDoubleOverflow) { - double lowest_double = std::numeric_limits::lowest(); - auto cel_value = CelValue::CreateDouble(lowest_double); - - // Double exceeds float precision, overflow to -infinity. - FloatValue wrapper; - wrapper.set_value(-std::numeric_limits::infinity()); - ExpectWrappedMessage(cel_value, wrapper); - - double max_double = std::numeric_limits::max(); - cel_value = CelValue::CreateDouble(max_double); - - wrapper.set_value(std::numeric_limits::infinity()); - ExpectWrappedMessage(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapInt64) { - int32_t num = std::numeric_limits::lowest(); - auto cel_value = CelValue::CreateInt64(num); - - Value json; - json.set_number_value(static_cast(num)); - ExpectWrappedMessage(cel_value, json); - - Int64Value wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapInt64ToInt32Value) { - int32_t num = std::numeric_limits::lowest(); - auto cel_value = CelValue::CreateInt64(num); - - Int32Value wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapFailureInt64ToInt32Value) { - int64_t num = std::numeric_limits::lowest(); - auto cel_value = CelValue::CreateInt64(num); - - Int32Value wrapper; - ExpectNotWrapped(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapInt64ToValue) { - int64_t max = std::numeric_limits::max(); - auto cel_value = CelValue::CreateInt64(max); - - Value json; - json.set_string_value(absl::StrCat(max)); - ExpectWrappedMessage(cel_value, json); - - int64_t min = std::numeric_limits::min(); - cel_value = CelValue::CreateInt64(min); - - json.set_string_value(absl::StrCat(min)); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapUint64) { - uint32_t num = std::numeric_limits::max(); - auto cel_value = CelValue::CreateUint64(num); - - Value json; - json.set_number_value(static_cast(num)); - ExpectWrappedMessage(cel_value, json); - - UInt64Value wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapUint64ToUint32Value) { - uint32_t num = std::numeric_limits::max(); - auto cel_value = CelValue::CreateUint64(num); - - UInt32Value wrapper; - wrapper.set_value(num); - ExpectWrappedMessage(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapUint64ToValue) { - uint64_t num = std::numeric_limits::max(); - auto cel_value = CelValue::CreateUint64(num); - - Value json; - json.set_string_value(absl::StrCat(num)); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapFailureUint64ToUint32Value) { - uint64_t num = std::numeric_limits::max(); - auto cel_value = CelValue::CreateUint64(num); - - UInt32Value wrapper; - ExpectNotWrapped(cel_value, wrapper); -} - -TEST_F(CelProtoWrapperTest, WrapString) { - std::string str = "test"; - auto cel_value = CelValue::CreateString(CelValue::StringHolder(&str)); - - Value json; - json.set_string_value(str); - ExpectWrappedMessage(cel_value, json); - - StringValue wrapper; - wrapper.set_value(str); - ExpectWrappedMessage(cel_value, wrapper); - - Any any; - any.PackFrom(wrapper); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapTimestamp) { - absl::Time ts = absl::FromUnixSeconds(1615852799); - auto cel_value = CelValue::CreateTimestamp(ts); - - Timestamp t; - t.set_seconds(1615852799); - ExpectWrappedMessage(cel_value, t); - - Any any; - any.PackFrom(t); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapTimestampToValue) { - absl::Time ts = absl::FromUnixSeconds(1615852799); - auto cel_value = CelValue::CreateTimestamp(ts); - - Value json; - json.set_string_value("2021-03-15T23:59:59Z"); - ExpectWrappedMessage(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapList) { - std::vector list_elems = { - CelValue::CreateDouble(1.5), - CelValue::CreateInt64(-2L), - }; - ContainerBackedListImpl list(std::move(list_elems)); - auto cel_value = CelValue::CreateList(&list); - - Value json; - json.mutable_list_value()->add_values()->set_number_value(1.5); - json.mutable_list_value()->add_values()->set_number_value(-2.); - ExpectWrappedMessage(cel_value, json); - ExpectWrappedMessage(cel_value, json.list_value()); - - Any any; - any.PackFrom(json.list_value()); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapFailureListValueBadJSON) { - TestMessage message; - std::vector list_elems = { - CelValue::CreateDouble(1.5), - UnwrapMessageToValue(&message, &ProtobufValueFactoryImpl, arena()), - }; - ContainerBackedListImpl list(std::move(list_elems)); - auto cel_value = CelValue::CreateList(&list); - - Value json; - ExpectNotWrapped(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapStruct) { - const std::string kField1 = "field1"; - std::vector> args = { - {CelValue::CreateString(CelValue::StringHolder(&kField1)), - CelValue::CreateBool(true)}}; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr cel_map, - CreateContainerBackedMap( - absl::Span>(args.data(), args.size()))); - auto cel_value = CelValue::CreateMap(cel_map.get()); - - Value json; - (*json.mutable_struct_value()->mutable_fields())[kField1].set_bool_value( - true); - ExpectWrappedMessage(cel_value, json); - ExpectWrappedMessage(cel_value, json.struct_value()); - - Any any; - any.PackFrom(json.struct_value()); - ExpectWrappedMessage(cel_value, any); -} - -TEST_F(CelProtoWrapperTest, WrapFailureStructBadKeyType) { - std::vector> args = { - {CelValue::CreateInt64(1L), CelValue::CreateBool(true)}}; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr cel_map, - CreateContainerBackedMap( - absl::Span>(args.data(), args.size()))); - auto cel_value = CelValue::CreateMap(cel_map.get()); - - Value json; - ExpectNotWrapped(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapFailureStructBadValueType) { - const std::string kField1 = "field1"; - TestMessage bad_value; - std::vector> args = { - {CelValue::CreateString(CelValue::StringHolder(&kField1)), - UnwrapMessageToValue(&bad_value, &ProtobufValueFactoryImpl, arena())}}; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr cel_map, - CreateContainerBackedMap( - absl::Span>(args.data(), args.size()))); - auto cel_value = CelValue::CreateMap(cel_map.get()); - Value json; - ExpectNotWrapped(cel_value, json); -} - -class TestMap : public CelMapBuilder { - public: - absl::StatusOr ListKeys() const override { - return absl::UnimplementedError("test"); - } -}; - -TEST_F(CelProtoWrapperTest, WrapFailureStructListKeysUnimplemented) { - const std::string kField1 = "field1"; - TestMap map; - ASSERT_OK(map.Add(CelValue::CreateString(CelValue::StringHolder(&kField1)), - CelValue::CreateString(CelValue::StringHolder(&kField1)))); - - auto cel_value = CelValue::CreateMap(&map); - Value json; - ExpectNotWrapped(cel_value, json); -} - -TEST_F(CelProtoWrapperTest, WrapFailureWrongType) { - auto cel_value = CelValue::CreateNull(); - std::vector wrong_types = { - &BoolValue::default_instance(), &BytesValue::default_instance(), - &DoubleValue::default_instance(), &Duration::default_instance(), - &FloatValue::default_instance(), &Int32Value::default_instance(), - &Int64Value::default_instance(), &ListValue::default_instance(), - &StringValue::default_instance(), &Struct::default_instance(), - &Timestamp::default_instance(), &UInt32Value::default_instance(), - &UInt64Value::default_instance(), - }; - for (const auto* wrong_type : wrong_types) { - ExpectNotWrapped(cel_value, *wrong_type); - } -} - -TEST_F(CelProtoWrapperTest, WrapFailureErrorToAny) { - auto cel_value = CreateNoSuchFieldError(arena(), "error_field"); - ExpectNotWrapped(cel_value, Any::default_instance()); -} - TEST_F(CelProtoWrapperTest, DebugString) { google::protobuf::Empty e; // Note: the value factory is trivial so the debug string for a message-typed diff --git a/eval/public/structs/cel_proto_wrap_value_to_message.cc b/eval/public/structs/cel_proto_wrap_value_to_message.cc new file mode 100644 index 000000000..0b04e93d2 --- /dev/null +++ b/eval/public/structs/cel_proto_wrap_value_to_message.cc @@ -0,0 +1,895 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" + +#include +#include +#include +#include +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/duration.pb.h" +#include "google/protobuf/struct.pb.h" +#include "google/protobuf/timestamp.pb.h" +#include "google/protobuf/wrappers.pb.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/cord.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" +#include "eval/public/cel_value.h" +#include "internal/overflow.h" +#include "internal/proto_time_encoding.h" +#include "internal/status_macros.h" +#include "internal/time.h" +#include "internal/well_known_types.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/json/json.h" +#include "google/protobuf/message.h" + +namespace google::api::expr::runtime::internal { + +namespace { + +using google::protobuf::BoolValue; +using google::protobuf::BytesValue; +using google::protobuf::DoubleValue; +using google::protobuf::Duration; +using google::protobuf::Int64Value; +using google::protobuf::ListValue; +using google::protobuf::StringValue; +using google::protobuf::Struct; +using google::protobuf::Timestamp; +using google::protobuf::UInt64Value; +using google::protobuf::Value; +using google::protobuf::Arena; +using google::protobuf::Descriptor; +using google::protobuf::Message; +using google::protobuf::MessageFactory; +using google::protobuf::json::MessageToJsonString; +using google::protobuf::json::PrintOptions; + +// kMaxIntJSON is defined as the Number.MAX_SAFE_INTEGER value per EcmaScript 6. +constexpr int64_t kMaxIntJSON = (1ll << 53) - 1; + +// kMinIntJSON is defined as the Number.MIN_SAFE_INTEGER value per EcmaScript 6. +constexpr int64_t kMinIntJSON = -kMaxIntJSON; + +// IsJSONSafe indicates whether the int is safely representable as a floating +// point value in JSON. +static bool IsJSONSafe(int64_t i) { + return i >= kMinIntJSON && i <= kMaxIntJSON; +} + +// IsJSONSafe indicates whether the uint is safely representable as a floating +// point value in JSON. +static bool IsJSONSafe(uint64_t i) { + return i <= static_cast(kMaxIntJSON); +} + +static bool IsEmptyProto(const google::protobuf::Descriptor* descriptor) { + return descriptor->full_name() == "google.protobuf.Empty"; +} + +static bool IsFieldMaskProto(const google::protobuf::Descriptor* descriptor) { + return descriptor->full_name() == "google.protobuf.FieldMask"; +} + +static std::optional GetFieldMaskJsonString( + const google::protobuf::Message& message) { + // TODO(b/540507668): Refactor to pipe descriptor_pool through + // ValueFromValue to use internal::MessageToJson. + PrintOptions json_options; + std::string json_str; + auto status = MessageToJsonString(message, &json_str, json_options); + if (!status.ok()) { + ABSL_LOG(ERROR) << "Failed to convert FieldMask to JSON: " << status; + return std::nullopt; + } + // If JSON marshalling is correct, we know we'll always get a plain + // JSON string value and it shouldn't contain any escapes that we need + // to interpret. + if (json_str.size() >= 2 && json_str.front() == '"' && + json_str.back() == '"') { + return json_str.substr(1, json_str.size() - 2); + } + return json_str; +} + +struct IgnoreErrorAndReturnNullptr { + std::nullptr_t operator()(const absl::Status& status) const { + status.IgnoreError(); + return nullptr; + } +}; + +google::protobuf::Message* DurationFromValue(const google::protobuf::Message* prototype, + const CelValue& value, + google::protobuf::Arena* arena) { + absl::Duration val; + if (!value.GetValue(&val)) { + return nullptr; + } + if (!cel::internal::ValidateDuration(val).ok()) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetDurationReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.UnsafeSetFromAbslDuration(message, val); + return message; +} + +google::protobuf::Message* BoolFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + bool val; + if (!value.GetValue(&val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetBoolValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, val); + return message; +} + +google::protobuf::Message* BytesFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + CelValue::BytesHolder view_val; + if (!value.GetValue(&view_val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetBytesValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, view_val.value()); + return message; +} + +google::protobuf::Message* DoubleFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + double val; + if (!value.GetValue(&val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetDoubleValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, val); + return message; +} + +google::protobuf::Message* FloatFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + double val; + if (!value.GetValue(&val)) { + return nullptr; + } + float fval = val; + // Abort the conversion if the value is outside the float range. + if (val > std::numeric_limits::max()) { + fval = std::numeric_limits::infinity(); + } else if (val < std::numeric_limits::lowest()) { + fval = -std::numeric_limits::infinity(); + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetFloatValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, static_cast(fval)); + return message; +} + +google::protobuf::Message* Int32FromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + int64_t val; + if (!value.GetValue(&val)) { + return nullptr; + } + if (!cel::internal::CheckedInt64ToInt32(val).ok()) { + return nullptr; + } + int32_t ival = static_cast(val); + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetInt32ValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, ival); + return message; +} + +google::protobuf::Message* Int64FromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + int64_t val; + if (!value.GetValue(&val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetInt64ValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, val); + return message; +} + +google::protobuf::Message* StringFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + CelValue::StringHolder view_val; + if (!value.GetValue(&view_val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetStringValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, view_val.value()); + return message; +} + +google::protobuf::Message* TimestampFromValue(const google::protobuf::Message* prototype, + const CelValue& value, + google::protobuf::Arena* arena) { + absl::Time val; + if (!value.GetValue(&val)) { + return nullptr; + } + if (!cel::internal::ValidateTimestamp(val).ok()) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetTimestampReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.UnsafeSetFromAbslTime(message, val); + return message; +} + +google::protobuf::Message* UInt32FromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + uint64_t val; + if (!value.GetValue(&val)) { + return nullptr; + } + if (!cel::internal::CheckedUint64ToUint32(val).ok()) { + return nullptr; + } + uint32_t ival = static_cast(val); + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetUInt32ValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, ival); + return message; +} + +google::protobuf::Message* UInt64FromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + uint64_t val; + if (!value.GetValue(&val)) { + return nullptr; + } + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetUInt64ValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetValue(message, val); + return message; +} + +google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, const CelValue& value, + google::protobuf::Arena* arena); + +google::protobuf::Message* ValueFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + return ValueFromValue(prototype->New(arena), value, arena); +} + +google::protobuf::Message* ListFromValue(google::protobuf::Message* message, const CelValue& value, + google::protobuf::Arena* arena) { + if (!value.IsList()) { + return nullptr; + } + const CelList& list = *value.ListOrDie(); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetListValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + for (int i = 0; i < list.size(); i++) { + auto e = list.Get(arena, i); + auto* elem = reflection.AddValues(message); + if (ValueFromValue(elem, e, arena) == nullptr) { + return nullptr; + } + } + return message; +} + +google::protobuf::Message* ListFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + if (!value.IsList()) { + return nullptr; + } + return ListFromValue(prototype->New(arena), value, arena); +} + +google::protobuf::Message* StructFromValue(google::protobuf::Message* message, + const CelValue& value, google::protobuf::Arena* arena) { + if (!value.IsMap()) { + return nullptr; + } + const CelMap& map = *value.MapOrDie(); + absl::StatusOr keys_or = map.ListKeys(arena); + if (!keys_or.ok()) { + // If map doesn't support listing keys, it can't pack into a Struct value. + // This will surface as a CEL error when the object creation expression + // fails. + return nullptr; + } + const CelList& keys = **keys_or; + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetStructReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + for (int i = 0; i < keys.size(); i++) { + auto k = keys.Get(arena, i); + // If the key is not a string type, abort the conversion. + if (!k.IsString()) { + return nullptr; + } + absl::string_view key = k.StringOrDie().value(); + + auto v = map.Get(arena, k); + if (!v.has_value()) { + return nullptr; + } + auto* field = reflection.InsertField(message, key); + if (ValueFromValue(field, *v, arena) == nullptr) { + return nullptr; + } + } + return message; +} + +google::protobuf::Message* StructFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + if (!value.IsMap()) { + return nullptr; + } + return StructFromValue(prototype->New(arena), value, arena); +} + +google::protobuf::Message* ValueFromValue(google::protobuf::Message* message, const CelValue& value, + google::protobuf::Arena* arena) { + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetValueReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + switch (value.type()) { + case CelValue::Type::kBool: { + bool val; + if (value.GetValue(&val)) { + reflection.SetBoolValue(message, val); + return message; + } + } break; + case CelValue::Type::kBytes: { + // Base64 encode byte strings to ensure they can safely be transported + // in a JSON string. + CelValue::BytesHolder val; + if (value.GetValue(&val)) { + reflection.SetStringValueFromBytes(message, val.value()); + return message; + } + } break; + case CelValue::Type::kDouble: { + double val; + if (value.GetValue(&val)) { + reflection.SetNumberValue(message, val); + return message; + } + } break; + case CelValue::Type::kDuration: { + // Convert duration values to a protobuf JSON format. + absl::Duration val; + if (value.GetValue(&val)) { + CEL_RETURN_IF_ERROR(cel::internal::ValidateDuration(val)) + .With(IgnoreErrorAndReturnNullptr()); + reflection.SetStringValueFromDuration(message, val); + return message; + } + } break; + case CelValue::Type::kInt64: { + int64_t val; + // Convert int64_t values within the int53 range to doubles, otherwise + // serialize the value to a string. + if (value.GetValue(&val)) { + reflection.SetNumberValue(message, val); + return message; + } + } break; + case CelValue::Type::kString: { + CelValue::StringHolder val; + if (value.GetValue(&val)) { + reflection.SetStringValue(message, val.value()); + return message; + } + } break; + case CelValue::Type::kTimestamp: { + // Convert timestamp values to a protobuf JSON format. + absl::Time val; + if (value.GetValue(&val)) { + CEL_RETURN_IF_ERROR(cel::internal::ValidateTimestamp(val)) + .With(IgnoreErrorAndReturnNullptr()); + reflection.SetStringValueFromTimestamp(message, val); + return message; + } + } break; + case CelValue::Type::kUint64: { + uint64_t val; + // Convert uint64_t values within the int53 range to doubles, otherwise + // serialize the value to a string. + if (value.GetValue(&val)) { + reflection.SetNumberValue(message, val); + return message; + } + } break; + case CelValue::Type::kList: { + if (ListFromValue(reflection.MutableListValue(message), value, arena) != + nullptr) { + return message; + } + } break; + case CelValue::Type::kMap: { + if (StructFromValue(reflection.MutableStructValue(message), value, + arena) != nullptr) { + return message; + } + } break; + case CelValue::Type::kMessage: { + const google::protobuf::Message* message_ptr = value.MessageOrDie(); + if (IsEmptyProto(message_ptr->GetDescriptor())) { + reflection.MutableStructValue(message); + return message; + } + if (IsFieldMaskProto(message_ptr->GetDescriptor())) { + std::optional fm_str = + GetFieldMaskJsonString(*message_ptr); + if (fm_str.has_value()) { + reflection.SetStringValue(message, *fm_str); + return message; + } + return nullptr; + } + return nullptr; + } break; + case CelValue::Type::kNullType: + reflection.SetNullValue(message); + return message; + break; + default: + return nullptr; + } + return nullptr; +} + +bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* arena); + +bool ListFromValue(ListValue* json_list, const CelValue& value, + google::protobuf::Arena* arena) { + if (!value.IsList()) { + return false; + } + const CelList& list = *value.ListOrDie(); + for (int i = 0; i < list.size(); i++) { + auto e = list.Get(arena, i); + Value* elem = json_list->add_values(); + if (!ValueFromValue(elem, e, arena)) { + return false; + } + } + return true; +} + +bool StructFromValue(Struct* json_struct, const CelValue& value, + google::protobuf::Arena* arena) { + if (!value.IsMap()) { + return false; + } + const CelMap& map = *value.MapOrDie(); + absl::StatusOr keys_or = map.ListKeys(arena); + if (!keys_or.ok()) { + // If map doesn't support listing keys, it can't pack into a Struct value. + // This will surface as a CEL error when the object creation expression + // fails. + return false; + } + const CelList& keys = **keys_or; + auto fields = json_struct->mutable_fields(); + for (int i = 0; i < keys.size(); i++) { + auto k = keys.Get(arena, i); + // If the key is not a string type, abort the conversion. + if (!k.IsString()) { + return false; + } + absl::string_view key = k.StringOrDie().value(); + + auto v = map.Get(arena, k); + if (!v.has_value()) { + return false; + } + Value field_value; + if (!ValueFromValue(&field_value, *v, arena)) { + return false; + } + (*fields)[std::string(key)] = field_value; + } + return true; +} + +bool ValueFromValue(Value* json, const CelValue& value, google::protobuf::Arena* arena) { + switch (value.type()) { + case CelValue::Type::kBool: { + bool val; + if (value.GetValue(&val)) { + json->set_bool_value(val); + return true; + } + } break; + case CelValue::Type::kBytes: { + // Base64 encode byte strings to ensure they can safely be transported + // in a JSON string. + CelValue::BytesHolder val; + if (value.GetValue(&val)) { + json->set_string_value(absl::Base64Escape(val.value())); + return true; + } + } break; + case CelValue::Type::kDouble: { + double val; + if (value.GetValue(&val)) { + json->set_number_value(val); + return true; + } + } break; + case CelValue::Type::kDuration: { + // Convert duration values to a protobuf JSON format. + absl::Duration val; + if (value.GetValue(&val)) { + auto encode = cel::internal::EncodeDurationToString(val); + if (!encode.ok()) { + return false; + } + json->set_string_value(*encode); + return true; + } + } break; + case CelValue::Type::kInt64: { + int64_t val; + // Convert int64_t values within the int53 range to doubles, otherwise + // serialize the value to a string. + if (value.GetValue(&val)) { + if (IsJSONSafe(val)) { + json->set_number_value(val); + } else { + json->set_string_value(absl::StrCat(val)); + } + return true; + } + } break; + case CelValue::Type::kString: { + CelValue::StringHolder val; + if (value.GetValue(&val)) { + json->set_string_value(val.value()); + return true; + } + } break; + case CelValue::Type::kTimestamp: { + // Convert timestamp values to a protobuf JSON format. + absl::Time val; + if (value.GetValue(&val)) { + auto encode = cel::internal::EncodeTimeToString(val); + if (!encode.ok()) { + return false; + } + json->set_string_value(*encode); + return true; + } + } break; + case CelValue::Type::kUint64: { + uint64_t val; + // Convert uint64_t values within the int53 range to doubles, otherwise + // serialize the value to a string. + if (value.GetValue(&val)) { + if (IsJSONSafe(val)) { + json->set_number_value(val); + } else { + json->set_string_value(absl::StrCat(val)); + } + return true; + } + } break; + case CelValue::Type::kList: + return ListFromValue(json->mutable_list_value(), value, arena); + case CelValue::Type::kMap: + return StructFromValue(json->mutable_struct_value(), value, arena); + case CelValue::Type::kMessage: { + const google::protobuf::Message* message_ptr = value.MessageOrDie(); + if (IsEmptyProto(message_ptr->GetDescriptor())) { + json->mutable_struct_value(); + return true; + } + if (IsFieldMaskProto(message_ptr->GetDescriptor())) { + std::optional fm_str = + GetFieldMaskJsonString(*message_ptr); + if (fm_str.has_value()) { + json->set_string_value(*fm_str); + return true; + } + return false; + } + return false; + } + case CelValue::Type::kNullType: + json->set_null_value(protobuf::NULL_VALUE); + return true; + default: + return false; + } + return false; +} + +google::protobuf::Message* AnyFromValue(const google::protobuf::Message* prototype, + const CelValue& value, google::protobuf::Arena* arena) { + std::string type_name; + absl::Cord payload; + + // In open source, any->PackFrom() returns void rather than boolean. + switch (value.type()) { + case CelValue::Type::kBool: { + BoolValue v; + type_name = v.GetTypeName(); + v.set_value(value.BoolOrDie()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kBytes: { + BytesValue v; + type_name = v.GetTypeName(); + v.set_value(value.BytesOrDie().value()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kDouble: { + DoubleValue v; + type_name = v.GetTypeName(); + v.set_value(value.DoubleOrDie()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kDuration: { + Duration v; + if (!cel::internal::EncodeDuration(value.DurationOrDie(), &v).ok()) { + return nullptr; + } + type_name = v.GetTypeName(); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kInt64: { + Int64Value v; + type_name = v.GetTypeName(); + v.set_value(value.Int64OrDie()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kString: { + StringValue v; + type_name = v.GetTypeName(); + v.set_value(std::string(value.StringOrDie().value())); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kTimestamp: { + Timestamp v; + if (!cel::internal::EncodeTime(value.TimestampOrDie(), &v).ok()) { + return nullptr; + } + type_name = v.GetTypeName(); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kUint64: { + UInt64Value v; + type_name = v.GetTypeName(); + v.set_value(value.Uint64OrDie()); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kList: { + ListValue v; + if (!ListFromValue(&v, value, arena)) { + return nullptr; + } + type_name = v.GetTypeName(); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kMap: { + Struct v; + if (!StructFromValue(&v, value, arena)) { + return nullptr; + } + type_name = v.GetTypeName(); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kNullType: { + Value v; + type_name = v.GetTypeName(); + v.set_null_value(google::protobuf::NULL_VALUE); + payload = v.SerializeAsCord(); + } break; + case CelValue::Type::kMessage: { + type_name = value.MessageWrapperOrDie().message_ptr()->GetTypeName(); + payload = value.MessageWrapperOrDie().message_ptr()->SerializeAsCord(); + } break; + default: + return nullptr; + } + + auto* message = prototype->New(arena); + CEL_ASSIGN_OR_RETURN( + auto reflection, + cel::well_known_types::GetAnyReflection(message->GetDescriptor()), + _.With(IgnoreErrorAndReturnNullptr())); + reflection.SetTypeUrl(message, + absl::StrCat("type.googleapis.com/", type_name)); + reflection.SetValue(message, payload); + return message; +} + +bool IsAlreadyWrapped(google::protobuf::Descriptor::WellKnownType wkt, + const CelValue& value) { + if (value.IsMessage()) { + const auto* msg = value.MessageOrDie(); + if (wkt == msg->GetDescriptor()->well_known_type()) { + return true; + } + } + return false; +} + +// MessageFromValueMaker makes a specific protobuf Message instance based on +// the desired protobuf type name and an input CelValue. +// +// It holds a registry of CelValue factories for specific subtypes of Message. +// If message does not match any of types stored in registry, an the factory +// returns an absent value. +class MessageFromValueMaker { + public: + // Non-copyable, non-assignable + MessageFromValueMaker(const MessageFromValueMaker&) = delete; + MessageFromValueMaker& operator=(const MessageFromValueMaker&) = delete; + + static google::protobuf::Message* MaybeWrapMessage(const google::protobuf::Descriptor* descriptor, + google::protobuf::MessageFactory* factory, + const CelValue& value, + Arena* arena) { + switch (descriptor->well_known_type()) { + case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return DoubleFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return FloatFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return Int64FromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return UInt64FromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return Int32FromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return UInt32FromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return StringFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return BytesFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_BOOLVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return BoolFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_ANY: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return AnyFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return DurationFromValue(factory->GetPrototype(descriptor), value, + arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return TimestampFromValue(factory->GetPrototype(descriptor), value, + arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return ValueFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_LISTVALUE: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return ListFromValue(factory->GetPrototype(descriptor), value, arena); + case google::protobuf::Descriptor::WELLKNOWNTYPE_STRUCT: + if (IsAlreadyWrapped(descriptor->well_known_type(), value)) { + return nullptr; + } + return StructFromValue(factory->GetPrototype(descriptor), value, arena); + // WELLKNOWNTYPE_FIELDMASK has no special CelValue type + default: + return nullptr; + } + } +}; + +} // namespace + +const google::protobuf::Message* MaybeWrapValueToMessage( + const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, + const CelValue& value, Arena* arena) { + google::protobuf::Message* msg = MessageFromValueMaker::MaybeWrapMessage( + descriptor, factory, value, arena); + return msg; +} + +} // namespace google::api::expr::runtime::internal diff --git a/eval/public/structs/cel_proto_wrap_value_to_message.h b/eval/public/structs/cel_proto_wrap_value_to_message.h new file mode 100644 index 000000000..9bc8581e1 --- /dev/null +++ b/eval/public/structs/cel_proto_wrap_value_to_message.h @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAP_VALUE_TO_MESSAGE_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAP_VALUE_TO_MESSAGE_H_ + +#include "eval/public/cel_value.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" + +namespace google::api::expr::runtime::internal { + +// MaybeWrapValue attempts to wrap the input value in a proto message with +// the given type_name. If the value can be wrapped, it is returned as a +// protobuf message. Otherwise, the result will be nullptr. +// +// This method is the complement to UnwrapMessageToValue which may unwrap a +// protobuf message to native CelValue representation during a protobuf field +// read. +// Just as CreateMessage should only be used when reading protobuf values, +// MaybeWrapValueToMessage should only be used when assigning protobuf fields. +const google::protobuf::Message* MaybeWrapValueToMessage( + const google::protobuf::Descriptor* descriptor, google::protobuf::MessageFactory* factory, + const CelValue& value, google::protobuf::Arena* arena); + +} // namespace google::api::expr::runtime::internal + +#endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_CEL_PROTO_WRAP_VALUE_TO_MESSAGE_H_ diff --git a/eval/public/structs/cel_proto_wrap_value_to_message_test.cc b/eval/public/structs/cel_proto_wrap_value_to_message_test.cc new file mode 100644 index 000000000..c05d1203d --- /dev/null +++ b/eval/public/structs/cel_proto_wrap_value_to_message_test.cc @@ -0,0 +1,616 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" + +#include +#include +#include +#include +#include +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/duration.pb.h" +#include "google/protobuf/empty.pb.h" +#include "google/protobuf/field_mask.pb.h" +#include "google/protobuf/struct.pb.h" +#include "google/protobuf/timestamp.pb.h" +#include "google/protobuf/wrappers.pb.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" +#include "absl/time/time.h" +#include "absl/types/span.h" +#include "eval/public/cel_value.h" +#include "eval/public/containers/container_backed_list_impl.h" +#include "eval/public/containers/container_backed_map_impl.h" +#include "eval/public/structs/trivial_legacy_type_info.h" +#include "eval/testutil/test_message.pb.h" +#include "internal/testing.h" +#include "testutil/util.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/dynamic_message.h" +#include "google/protobuf/message.h" +#include "google/protobuf/text_format.h" + +namespace google::api::expr::runtime::internal { + +namespace { + +using google::protobuf::Any; +using google::protobuf::BoolValue; +using google::protobuf::BytesValue; +using google::protobuf::DoubleValue; +using google::protobuf::Duration; +using google::protobuf::FieldMask; +using google::protobuf::FloatValue; +using google::protobuf::Int32Value; +using google::protobuf::Int64Value; +using google::protobuf::ListValue; +using google::protobuf::StringValue; +using google::protobuf::Struct; +using google::protobuf::Timestamp; +using google::protobuf::UInt32Value; +using google::protobuf::UInt64Value; +using google::protobuf::Value; +using google::protobuf::Arena; +using ::google::protobuf::TextFormat; + +CelValue MessageToCelValue(const google::protobuf::Message* m) { + return CelValue::CreateMessageWrapper( + CelValue::MessageWrapper(m, TrivialTypeInfo::GetInstance())); +} + +class CelProtoWrapValueToMessageTest : public ::testing::Test { + protected: + CelProtoWrapValueToMessageTest() = default; + + void ExpectWrappedMessage(const CelValue& value, + const google::protobuf::Message& message) { + // Test the input value wraps to the destination message type. + auto* result = MaybeWrapValueToMessage( + message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), + value, arena()); + EXPECT_TRUE(result != nullptr); + EXPECT_THAT(result, testutil::EqualsProto(message)); + + // Ensure that double wrapping results in the object being wrapped once. + auto* identity = MaybeWrapValueToMessage( + message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), + MessageToCelValue(result), arena()); + EXPECT_TRUE(identity == nullptr); + + // Check to make sure that even dynamic messages can be used as input to + // the wrapping call. + result = MaybeWrapValueToMessage( + ReflectedCopy(message)->GetDescriptor(), + ReflectedCopy(message)->GetReflection()->GetMessageFactory(), value, + arena()); + EXPECT_TRUE(result != nullptr); + EXPECT_THAT(result, testutil::EqualsProto(message)); + } + + void ExpectNotWrapped(const CelValue& value, const google::protobuf::Message& message) { + // Test the input value does not wrap by asserting value == result. + auto result = MaybeWrapValueToMessage( + message.GetDescriptor(), message.GetReflection()->GetMessageFactory(), + value, arena()); + EXPECT_TRUE(result == nullptr); + } + + std::unique_ptr ReflectedCopy( + const google::protobuf::Message& message) { + std::unique_ptr dynamic_value( + factory_.GetPrototype(message.GetDescriptor())->New()); + dynamic_value->CopyFrom(message); + return dynamic_value; + } + + Arena* arena() { return &arena_; } + + private: + Arena arena_; + google::protobuf::DynamicMessageFactory factory_; +}; + +TEST_F(CelProtoWrapValueToMessageTest, WrapNull) { + auto cel_value = CelValue::CreateNull(); + + Value json; + json.set_null_value(protobuf::NULL_VALUE); + ExpectWrappedMessage(cel_value, json); + + Any any; + any.PackFrom(json); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapBool) { + auto cel_value = CelValue::CreateBool(true); + + Value json; + json.set_bool_value(true); + ExpectWrappedMessage(cel_value, json); + + BoolValue wrapper; + wrapper.set_value(true); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapBytes) { + std::string str = "hello world"; + auto cel_value = CelValue::CreateBytes(CelValue::BytesHolder(&str)); + + BytesValue wrapper; + wrapper.set_value(str); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapBytesToValue) { + std::string str = "hello world"; + auto cel_value = CelValue::CreateBytes(CelValue::BytesHolder(&str)); + + Value json; + json.set_string_value("aGVsbG8gd29ybGQ="); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDuration) { + auto cel_value = CelValue::CreateDuration(absl::Seconds(300)); + + Duration d; + d.set_seconds(300); + ExpectWrappedMessage(cel_value, d); + + Any any; + any.PackFrom(d); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDurationToValue) { + auto cel_value = CelValue::CreateDuration(absl::Seconds(300)); + + Value json; + json.set_string_value("300s"); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDouble) { + double num = 1.5; + auto cel_value = CelValue::CreateDouble(num); + + Value json; + json.set_number_value(num); + ExpectWrappedMessage(cel_value, json); + + DoubleValue wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDoubleToFloatValue) { + double num = 1.5; + auto cel_value = CelValue::CreateDouble(num); + + FloatValue wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); + + // Imprecise double -> float representation results in truncation. + double small_num = -9.9e-100; + wrapper.set_value(small_num); + cel_value = CelValue::CreateDouble(small_num); + ExpectWrappedMessage(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapDoubleOverflow) { + double lowest_double = std::numeric_limits::lowest(); + auto cel_value = CelValue::CreateDouble(lowest_double); + + // Double exceeds float precision, overflow to -infinity. + FloatValue wrapper; + wrapper.set_value(-std::numeric_limits::infinity()); + ExpectWrappedMessage(cel_value, wrapper); + + double max_double = std::numeric_limits::max(); + cel_value = CelValue::CreateDouble(max_double); + + wrapper.set_value(std::numeric_limits::infinity()); + ExpectWrappedMessage(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapInt64) { + int32_t num = std::numeric_limits::lowest(); + auto cel_value = CelValue::CreateInt64(num); + + Value json; + json.set_number_value(static_cast(num)); + ExpectWrappedMessage(cel_value, json); + + Int64Value wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapInt64ToInt32Value) { + int32_t num = std::numeric_limits::lowest(); + auto cel_value = CelValue::CreateInt64(num); + + Int32Value wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureInt64ToInt32Value) { + int64_t num = std::numeric_limits::lowest(); + auto cel_value = CelValue::CreateInt64(num); + + Int32Value wrapper; + ExpectNotWrapped(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapInt64ToValue) { + int64_t max = std::numeric_limits::max(); + auto cel_value = CelValue::CreateInt64(max); + + Value json; + json.set_string_value(absl::StrCat(max)); + ExpectWrappedMessage(cel_value, json); + + int64_t min = std::numeric_limits::min(); + cel_value = CelValue::CreateInt64(min); + + json.set_string_value(absl::StrCat(min)); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapUint64) { + uint32_t num = std::numeric_limits::max(); + auto cel_value = CelValue::CreateUint64(num); + + Value json; + json.set_number_value(static_cast(num)); + ExpectWrappedMessage(cel_value, json); + + UInt64Value wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapUint64ToUint32Value) { + uint32_t num = std::numeric_limits::max(); + auto cel_value = CelValue::CreateUint64(num); + + UInt32Value wrapper; + wrapper.set_value(num); + ExpectWrappedMessage(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapUint64ToValue) { + uint64_t num = std::numeric_limits::max(); + auto cel_value = CelValue::CreateUint64(num); + + Value json; + json.set_string_value(absl::StrCat(num)); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureUint64ToUint32Value) { + uint64_t num = std::numeric_limits::max(); + auto cel_value = CelValue::CreateUint64(num); + + UInt32Value wrapper; + ExpectNotWrapped(cel_value, wrapper); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapString) { + std::string str = "test"; + auto cel_value = CelValue::CreateString(CelValue::StringHolder(&str)); + + Value json; + json.set_string_value(str); + ExpectWrappedMessage(cel_value, json); + + StringValue wrapper; + wrapper.set_value(str); + ExpectWrappedMessage(cel_value, wrapper); + + Any any; + any.PackFrom(wrapper); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapTimestamp) { + absl::Time ts = absl::FromUnixSeconds(1615852799); + auto cel_value = CelValue::CreateTimestamp(ts); + + Timestamp t; + t.set_seconds(1615852799); + ExpectWrappedMessage(cel_value, t); + + Any any; + any.PackFrom(t); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapTimestampToValue) { + absl::Time ts = absl::FromUnixSeconds(1615852799); + auto cel_value = CelValue::CreateTimestamp(ts); + + Value json; + json.set_string_value("2021-03-15T23:59:59Z"); + ExpectWrappedMessage(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapList) { + std::vector list_elems = { + CelValue::CreateDouble(1.5), + CelValue::CreateInt64(-2L), + }; + ContainerBackedListImpl list(std::move(list_elems)); + auto cel_value = CelValue::CreateList(&list); + + Value json; + json.mutable_list_value()->add_values()->set_number_value(1.5); + json.mutable_list_value()->add_values()->set_number_value(-2.); + ExpectWrappedMessage(cel_value, json); + ExpectWrappedMessage(cel_value, json.list_value()); + + Any any; + any.PackFrom(json.list_value()); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureListValueBadJSON) { + TestMessage message; + std::vector list_elems = { + CelValue::CreateDouble(1.5), + MessageToCelValue(&message), + }; + ContainerBackedListImpl list(std::move(list_elems)); + auto cel_value = CelValue::CreateList(&list); + + Value json; + ExpectNotWrapped(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapStruct) { + const std::string kField1 = "field1"; + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField1)), + CelValue::CreateBool(true)}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Value json; + (*json.mutable_struct_value()->mutable_fields())[kField1].set_bool_value( + true); + ExpectWrappedMessage(cel_value, json); + ExpectWrappedMessage(cel_value, json.struct_value()); + + Any any; + any.PackFrom(json.struct_value()); + ExpectWrappedMessage(cel_value, any); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureStructBadKeyType) { + std::vector> args = { + {CelValue::CreateInt64(1L), CelValue::CreateBool(true)}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Value json; + ExpectNotWrapped(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureStructBadValueType) { + const std::string kField1 = "field1"; + TestMessage bad_value; + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField1)), + MessageToCelValue(&bad_value)}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + Value json; + ExpectNotWrapped(cel_value, json); +} + +class TestMap : public CelMapBuilder { + public: + absl::StatusOr ListKeys() const override { + return absl::UnimplementedError("test"); + } +}; + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureStructListKeysUnimplemented) { + const std::string kField1 = "field1"; + TestMap map; + ASSERT_THAT(map.Add(CelValue::CreateString(CelValue::StringHolder(&kField1)), + CelValue::CreateString(CelValue::StringHolder(&kField1))), + absl_testing::IsOk()); + + auto cel_value = CelValue::CreateMap(&map); + Value json; + ExpectNotWrapped(cel_value, json); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureWrongType) { + auto cel_value = CelValue::CreateNull(); + std::vector wrong_types = { + &BoolValue::default_instance(), &BytesValue::default_instance(), + &DoubleValue::default_instance(), &Duration::default_instance(), + &FloatValue::default_instance(), &Int32Value::default_instance(), + &Int64Value::default_instance(), &ListValue::default_instance(), + &StringValue::default_instance(), &Struct::default_instance(), + &Timestamp::default_instance(), &UInt32Value::default_instance(), + &UInt64Value::default_instance(), + }; + for (const auto* wrong_type : wrong_types) { + ExpectNotWrapped(cel_value, *wrong_type); + } +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFailureErrorToAny) { + auto cel_value = CreateNoSuchFieldError(arena(), "error_field"); + ExpectNotWrapped(cel_value, Any::default_instance()); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapFieldMaskToValue) { + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = MessageToCelValue(&field_mask); + + Value expected_message; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb(string_value: "foo.bar,baz")pb", + &expected_message)); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapMapWithFieldMaskToAny) { + const std::string kField = "field_mask"; + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = MessageToCelValue(&field_mask); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + CelValue cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + ASSERT_TRUE( + TextFormat::ParseFromString(R"pb( + fields { + key: "field_mask" + value { string_value: "foo.bar,baz" } + } + )pb", + &expected_struct)); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_struct)); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapListWithFieldMaskToAny) { + FieldMask field_mask; + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + paths: "foo.bar" paths: "baz" + )pb", + &field_mask)); + CelValue value = MessageToCelValue(&field_mask); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + ASSERT_TRUE(TextFormat::ParseFromString( + R"pb(values { string_value: "foo.bar,baz" })pb", &expected_list)); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_list)); + + ExpectWrappedMessage(list_value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapEmptyToValue) { + google::protobuf::Empty empty; + CelValue value = MessageToCelValue(&empty); + + Value expected_message; + expected_message.mutable_struct_value(); + + ExpectWrappedMessage(value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapMapWithEmptyToAny) { + const std::string kField = "empty"; + google::protobuf::Empty empty; + CelValue value = MessageToCelValue(&empty); + + std::vector> args = { + {CelValue::CreateString(CelValue::StringHolder(&kField)), value}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr cel_map, + CreateContainerBackedMap( + absl::Span>(args.data(), args.size()))); + auto cel_value = CelValue::CreateMap(cel_map.get()); + + Struct expected_struct; + (*expected_struct.mutable_fields())[kField].mutable_struct_value(); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_struct)); + + ExpectWrappedMessage(cel_value, expected_message); +} + +TEST_F(CelProtoWrapValueToMessageTest, WrapListWithEmptyToAny) { + google::protobuf::Empty empty; + CelValue value = MessageToCelValue(&empty); + + std::vector list_entries = {value}; + ContainerBackedListImpl cel_list(list_entries); + CelValue list_value = CelValue::CreateList(&cel_list); + + ListValue expected_list; + expected_list.add_values()->mutable_struct_value(); + Any expected_message; + ASSERT_TRUE(expected_message.PackFrom(expected_list)); + + ExpectWrappedMessage(list_value, expected_message); +} + +} // namespace + +} // namespace google::api::expr::runtime::internal diff --git a/eval/public/structs/cel_proto_wrapper.cc b/eval/public/structs/cel_proto_wrapper.cc index e220b64dd..8b85aafc2 100644 --- a/eval/public/structs/cel_proto_wrapper.cc +++ b/eval/public/structs/cel_proto_wrapper.cc @@ -26,7 +26,7 @@ #include "common/value.h" #include "eval/public/cel_value.h" #include "eval/public/message_wrapper.h" -#include "eval/public/structs/cel_proto_wrap_util.h" +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" #include "eval/public/structs/proto_message_type_adapter.h" #include "eval/public/structs/trivial_legacy_type_info_internal.h" #include "google/protobuf/arena.h" diff --git a/eval/public/structs/field_access_impl.cc b/eval/public/structs/field_access_impl.cc index 2bd9fff9d..652bcf778 100644 --- a/eval/public/structs/field_access_impl.cc +++ b/eval/public/structs/field_access_impl.cc @@ -29,6 +29,7 @@ #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "eval/public/structs/cel_proto_wrap_util.h" +#include "eval/public/structs/cel_proto_wrap_value_to_message.h" #include "internal/casts.h" #include "internal/overflow.h" #include "google/protobuf/arena.h" From af2375ac9a83a93717b5aaa3dffe814ebb586f4b Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 9 Sep 2026 13:20:11 -0700 Subject: [PATCH 211/350] Address portability problems closes #2253 PiperOrigin-RevId: 978716725 --- common/internal/byte_string.cc | 4 +++- common/internal/byte_string_test.cc | 6 ++++++ common/value.h | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/common/internal/byte_string.cc b/common/internal/byte_string.cc index d35c0efa2..201842a10 100644 --- a/common/internal/byte_string.cc +++ b/common/internal/byte_string.cc @@ -946,7 +946,9 @@ void ByteString::SetSmall(google::protobuf::Arena* absl_nullable arena, rep_.header.kind = ByteStringKind::kSmall; rep_.small.size = string.size(); rep_.small.arena = arena; - std::memcpy(rep_.small.data, string.data(), rep_.small.size); + if (!string.empty()) { + std::memcpy(rep_.small.data, string.data(), rep_.small.size); + } } void ByteString::SetSmall(google::protobuf::Arena* absl_nullable arena, diff --git a/common/internal/byte_string_test.cc b/common/internal/byte_string_test.cc index 902a2e877..c4ae0eae6 100644 --- a/common/internal/byte_string_test.cc +++ b/common/internal/byte_string_test.cc @@ -124,6 +124,12 @@ TEST_P(ByteStringTest, Default) { EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); } +TEST_P(ByteStringTest, ConstructNullDataStringView) { + ByteString byte_string(GetAllocator(), absl::string_view()); + EXPECT_THAT(byte_string, IsEmpty()); + EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); +} + TEST_P(ByteStringTest, ConstructSmallCString) { ByteString byte_string = ByteString(GetAllocator(), GetSmallString().c_str()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); diff --git a/common/value.h b/common/value.h index 34b4714a7..c2ca1c608 100644 --- a/common/value.h +++ b/common/value.h @@ -2802,7 +2802,7 @@ absl::StatusOr> StructValueMixin::Qualify( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { - ABSL_DCHECK_GT(qualifiers.size(), 0); + ABSL_DCHECK(!qualifiers.empty()); ABSL_DCHECK(descriptor_pool != nullptr); ABSL_DCHECK(message_factory != nullptr); ABSL_DCHECK(arena != nullptr); From f291946837b5f2f4db67345ae18fd393f8616983 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 9 Sep 2026 20:49:24 -0700 Subject: [PATCH 212/350] No public description PiperOrigin-RevId: 978916525 --- checker/internal/type_checker_builder_impl.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/checker/internal/type_checker_builder_impl.cc b/checker/internal/type_checker_builder_impl.cc index 4289fb528..ab0caa713 100644 --- a/checker/internal/type_checker_builder_impl.cc +++ b/checker/internal/type_checker_builder_impl.cc @@ -245,14 +245,16 @@ absl::Status ValidateType(const Type& t, bool check_type_param_name, return absl::OkStatus(); } case TypeKind::kOpaque: { - for (Type type_param : t.AsOpaque()->GetParameters()) { + OpaqueType opaque_type = t.GetOpaque(); + for (Type type_param : opaque_type.GetParameters()) { CEL_RETURN_IF_ERROR(ValidateType(type_param, check_type_param_name, depth_limit, remaining_depth)); } return absl::OkStatus(); } case TypeKind::kType: { - for (Type type_param : t.AsType()->GetParameters()) { + TypeType type_type = t.GetType(); + for (Type type_param : type_type.GetParameters()) { CEL_RETURN_IF_ERROR(ValidateType(type_param, check_type_param_name, depth_limit, remaining_depth)); } From 994cdd58d9401a04f5880e66634b01ba81639e67 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 10 Sep 2026 11:50:15 -0700 Subject: [PATCH 213/350] Add checks against accidentally depending on non-hermetic message factories when copying messages. Avoids unexpected use-after-free bugs when messages are extended by an overlaid descriptor pool. PiperOrigin-RevId: 979308985 --- common/values/struct_value_builder.cc | 14 +- eval/eval/select_step.cc | 112 ++++++++++---- extensions/protobuf/BUILD | 1 + extensions/protobuf/value.h | 112 ++++++++++---- extensions/protobuf/value_test.cc | 146 ++++++++++++++++++ internal/well_known_types.cc | 6 + runtime/runtime.h | 17 +- runtime/runtime_builder_factory.h | 10 +- testutil/BUILD | 46 ++++++ testutil/test_external_extensions.proto | 42 +++++ ...test_external_extensions_descriptor_set.cc | 62 ++++++++ .../test_external_extensions_descriptor_set.h | 28 ++++ ...external_extensions_descriptor_set_test.cc | 56 +++++++ 13 files changed, 577 insertions(+), 75 deletions(-) create mode 100644 testutil/test_external_extensions.proto create mode 100644 testutil/test_external_extensions_descriptor_set.cc create mode 100644 testutil/test_external_extensions_descriptor_set.h create mode 100644 testutil/test_external_extensions_descriptor_set_test.cc diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index f43369e33..cafdbeb54 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -84,8 +85,17 @@ absl::StatusOr> ProtoMessageCopy( const google::protobuf::Message* absl_nonnull from_message) { CEL_ASSIGN_OR_RETURN(const auto* from_descriptor, GetDescriptor(*from_message)); - if (to_descriptor == from_descriptor) { - // Same. + if (to_descriptor == from_descriptor && + to_message->GetReflection()->GetMessageFactory() == + from_message->GetReflection()->GetMessageFactory()) { + // Same type, use proto reflection copy. + // + // We use the slower serialization copy if the factory is different to avoid + // adding an implicit lifetime dependency on the other factory. + // + // This should only happen if the embedding application is calling the + // builder directly or attempting to set the field from an unsafe wrapped + // message. to_message->CopyFrom(*from_message); return std::nullopt; } diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index e7974496b..0adb77fd6 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -36,11 +36,9 @@ namespace { using ::cel::BoolValue; using ::cel::ErrorValue; -using ::cel::MapValue; using ::cel::OptionalValue; using ::cel::ProtoWrapperTypeOptions; using ::cel::StringValue; -using ::cel::StructValue; using ::cel::Value; using ::cel::ValueKind; @@ -424,6 +422,42 @@ class DirectSelectStep : public DirectExpressionStep { bool enable_optional_types_; }; +bool CheckAttributeTrail(const std::string& field, ExecutionFrame* frame) { + if (!frame->attribute_tracking_enabled()) { + return false; + } + AttributeTrail& attr = frame->value_stack().PeekAttribute(); + attr = attr.Step(&field); + + absl::optional marked_attribute_check = + CheckForMarkedAttributes(attr, *frame); + if (marked_attribute_check.has_value()) { + frame->value_stack().Peek() = std::move(marked_attribute_check).value(); + return true; + } + + return false; +} + +bool SupportsCachedFieldDescriptor( + const cel::ParsedMessageValue& parsed_message, + const google::protobuf::Descriptor* descriptor, + const google::protobuf::FieldDescriptor* field_descriptor) { + ABSL_DCHECK_EQ(field_descriptor->containing_type(), descriptor); + const google::protobuf::Descriptor* rt_descriptor = parsed_message.GetDescriptor(); + + if (rt_descriptor != descriptor) { + return false; + } + + // Caller should have already checked this. Crash here if not instead of + // making proto crash. + ABSL_DCHECK_EQ(rt_descriptor->file()->pool(), + field_descriptor->file()->pool()); + + return true; +} + class ProtoSelectStep : public SelectStep { public: ProtoSelectStep(StringValue value, int64_t expr_id, @@ -447,15 +481,21 @@ class ProtoSelectStep : public SelectStep { const Value& arg = frame->value_stack().Peek(); if (auto unwrapped = arg.AsParsedMessage(); - unwrapped.has_value() && unwrapped->GetDescriptor() == descriptor_) { - return EvaluateModernMessageGetField(frame, *unwrapped); + unwrapped.has_value() && + SupportsCachedFieldDescriptor(*unwrapped, descriptor_, + field_descriptor_)) { + return EvaluateMessageFieldGet(frame, *unwrapped); } else if (const google::protobuf::Message* legacy_message = cel::interop_internal::GetLegacyMessage(arg); - legacy_message != nullptr && - legacy_message->GetDescriptor() == descriptor_) { + frame->options().enable_use_new_field_select_implementation && + legacy_message != nullptr) { + auto parsed_message = cel::UnsafeParsedMessageValue(legacy_message); // A little unfortunate, but need to special case for legacy values so we // can minimize back and forth interop conversions. - return EvaluateLegacyMessageGetField(frame, legacy_message); + if (SupportsCachedFieldDescriptor(parsed_message, descriptor_, + field_descriptor_)) { + return EvaluateMessageFieldGet(frame, legacy_message); + } } // If we get an unexpected value type, fall back to the generic // implementation. @@ -463,34 +503,18 @@ class ProtoSelectStep : public SelectStep { } private: - absl::Status EvaluateModernMessageGetField( + absl::Status EvaluateMessageFieldGet( ExecutionFrame* frame, const cel::ParsedMessageValue& parsed_message) const; - absl::Status EvaluateLegacyMessageGetField( - ExecutionFrame* frame, const google::protobuf::Message* legacy_message) const; + absl::Status EvaluateMessageFieldGet( + ExecutionFrame* frame, + const google::protobuf::Message* absl_nonnull legacy_message) const; const google::protobuf::Descriptor* descriptor_; const google::protobuf::FieldDescriptor* field_descriptor_; }; -bool CheckAttributeTrail(const std::string& field, ExecutionFrame* frame) { - if (!frame->attribute_tracking_enabled()) { - return false; - } - AttributeTrail& attr = frame->value_stack().PeekAttribute(); - attr = attr.Step(&field); - - absl::optional marked_attribute_check = - CheckForMarkedAttributes(attr, *frame); - if (marked_attribute_check.has_value()) { - frame->value_stack().Peek() = std::move(marked_attribute_check).value(); - return true; - } - - return false; -} - -absl::Status ProtoSelectStep::EvaluateModernMessageGetField( +absl::Status ProtoSelectStep::EvaluateMessageFieldGet( ExecutionFrame* frame, const cel::ParsedMessageValue& parsed_message) const { if (CheckAttributeTrail(field_, frame)) { @@ -501,8 +525,10 @@ absl::Status ProtoSelectStep::EvaluateModernMessageGetField( frame->message_factory(), frame->arena(), &frame->value_stack().Peek()); } -absl::Status ProtoSelectStep::EvaluateLegacyMessageGetField( - ExecutionFrame* frame, const google::protobuf::Message* legacy_message) const { +absl::Status ProtoSelectStep::EvaluateMessageFieldGet( + ExecutionFrame* frame, + const google::protobuf::Message* absl_nonnull legacy_message) const { + ABSL_DCHECK(legacy_message != nullptr); if (CheckAttributeTrail(field_, frame)) { return absl::OkStatus(); } @@ -534,15 +560,19 @@ class ProtoHasStep : public SelectStep { const Value& arg = frame->value_stack().Peek(); if (auto unwrapped = arg.AsParsedMessage(); - unwrapped.has_value() && unwrapped->GetDescriptor() == descriptor_) { + unwrapped.has_value() && + SupportsCachedFieldDescriptor(*unwrapped, descriptor_, + field_descriptor_)) { return EvaluateHas(frame, *unwrapped); } else if (const google::protobuf::Message* legacy_message = cel::interop_internal::GetLegacyMessage(arg); - legacy_message != nullptr && - legacy_message->GetDescriptor() == descriptor_) { + legacy_message != nullptr) { cel::ParsedMessageValue parsed_message = cel::UnsafeParsedMessageValue(legacy_message); - return EvaluateHas(frame, parsed_message); + if (SupportsCachedFieldDescriptor(parsed_message, descriptor_, + field_descriptor_)) { + return EvaluateHas(frame, parsed_message); + } } // If we get an unexpected value type, fall back to the generic // implementation. @@ -608,6 +638,20 @@ absl::StatusOr> CreateTypedSelectStep( const google::protobuf::FieldDescriptor* field_descriptor = resolved_field.GetMessage().descriptor(); + if (field_descriptor->file()->pool() != descriptor->file()->pool()) { + // The field descriptor is not in the same pool as the operand type. + // (this should only happen if an overlay extends the type). + // + // We don't have a way to determine if a runtime message is compatible + // with the resolved extension and proto's reflection implementation may + // crash. + // + // Fallback to the generic implementation. + return CreateSelectStep(std::move(field), test_only, expr_id, + enable_wrapper_type_null_unboxing, + enable_optional_types); + } + if (test_only) { return std::make_unique( std::move(field), expr_id, enable_wrapper_type_null_unboxing, diff --git a/extensions/protobuf/BUILD b/extensions/protobuf/BUILD index d814057e7..002da4764 100644 --- a/extensions/protobuf/BUILD +++ b/extensions/protobuf/BUILD @@ -123,6 +123,7 @@ cc_test( "//common:value_kind", "//common:value_testing", "//internal:testing", + "//testutil:test_external_extensions_descriptor_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", diff --git a/extensions/protobuf/value.h b/extensions/protobuf/value.h index b7a654064..68049787f 100644 --- a/extensions/protobuf/value.h +++ b/extensions/protobuf/value.h @@ -40,6 +40,70 @@ namespace cel::extensions { +namespace extensions_internal { +template +absl::Status ProtoMessageFromValue(const cel::Value& value, + google::protobuf::Message& dest_message) { + const auto* dest_descriptor = dest_message.GetDescriptor(); + const google::protobuf::Message* src_message = nullptr; + if (auto legacy_struct_value = + cel::common_internal::AsLegacyStructValue(value); + legacy_struct_value) { + src_message = legacy_struct_value->message_ptr(); + } + if (auto parsed_message_value = value.AsParsedMessage(); + parsed_message_value) { + src_message = cel::to_address(*parsed_message_value); + } + + if (src_message == nullptr) { + return TypeConversionError(value.GetRuntimeType(), + MessageType(dest_descriptor)) + .NativeValue(); + } + + const auto* src_descriptor = src_message->GetDescriptor(); + if (dest_descriptor != src_descriptor) { + goto slow_path; + } + + if constexpr (!SkipCopyCheck::value) { + // Try to catch cases where we'll take an implicit dependency on a + // dynamic message factory. + // + // This isn't exhaustive, but correctly checking requires fully + // traversing the source message which will approach the cost of the + // serialization round trip. + if (dest_message.GetReflection()->GetMessageFactory() != + src_message->GetReflection()->GetMessageFactory()) { + goto slow_path; + } + } + + dest_message.CopyFrom(*src_message); + return absl::OkStatus(); + +slow_path: + if (dest_descriptor->full_name() != src_descriptor->full_name()) { + return TypeConversionError(value.GetRuntimeType(), + MessageType(dest_descriptor)) + .NativeValue(); + } + + absl::Cord serialized; + if (!src_message->SerializePartialToCord(&serialized)) { + return absl::UnknownError(absl::StrCat("failed to serialize message: ", + src_descriptor->full_name())); + } + if (!dest_message.ParsePartialFromCord(serialized)) { + return absl::UnknownError(absl::StrCat("failed to parse message: ", + dest_descriptor->full_name())); + } + return absl::OkStatus(); +} + +} // namespace extensions_internal + // Adapt a protobuf message to a cel::Value. // // Handles unwrapping message types with special meanings in CEL (WKTs). @@ -56,41 +120,23 @@ ProtoMessageToValue(T&& value, message_factory, arena); } +// Unwraps a protobuf message from a cel::Value. inline absl::Status ProtoMessageFromValue(const Value& value, google::protobuf::Message& dest_message) { - const auto* dest_descriptor = dest_message.GetDescriptor(); - const google::protobuf::Message* src_message = nullptr; - if (auto legacy_struct_value = - cel::common_internal::AsLegacyStructValue(value); - legacy_struct_value) { - src_message = legacy_struct_value->message_ptr(); - } - if (auto parsed_message_value = value.AsParsedMessage(); - parsed_message_value) { - src_message = cel::to_address(*parsed_message_value); - } - if (src_message != nullptr) { - const auto* src_descriptor = src_message->GetDescriptor(); - if (dest_descriptor == src_descriptor) { - dest_message.CopyFrom(*src_message); - return absl::OkStatus(); - } - if (dest_descriptor->full_name() == src_descriptor->full_name()) { - absl::Cord serialized; - if (!src_message->SerializePartialToCord(&serialized)) { - return absl::UnknownError(absl::StrCat("failed to serialize message: ", - src_descriptor->full_name())); - } - if (!dest_message.ParsePartialFromCord(serialized)) { - return absl::UnknownError(absl::StrCat("failed to parse message: ", - dest_descriptor->full_name())); - } - return absl::OkStatus(); - } - } - return TypeConversionError(value.GetRuntimeType(), - MessageType(dest_descriptor)) - .NativeValue(); + return extensions_internal::ProtoMessageFromValue( + value, dest_message); +} + +// Unwraps a protobuf message from a cel::Value without checking for the +// presence of extensions. +// +// Warning: This function can lead to subtle use after free bugs if the caller +// is not careful to ensure that the source and destination message were created +// in a compatible way and do not outlive any implicit dependencies. +inline absl::Status ProtoMessageFromValueUnsafe(const Value& value, + google::protobuf::Message& dest_message) { + return extensions_internal::ProtoMessageFromValue( + value, dest_message); } } // namespace cel::extensions diff --git a/extensions/protobuf/value_test.cc b/extensions/protobuf/value_test.cc index 20d9dce2f..cc89588f7 100644 --- a/extensions/protobuf/value_test.cc +++ b/extensions/protobuf/value_test.cc @@ -37,7 +37,11 @@ #include "common/value_kind.h" #include "common/value_testing.h" #include "internal/testing.h" +#include "testutil/test_external_extensions_descriptor_set.h" #include "cel/expr/conformance/proto2/test_all_types.pb.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/dynamic_message.h" +#include "google/protobuf/message.h" #include "google/protobuf/text_format.h" namespace cel::extensions { @@ -796,5 +800,147 @@ TEST_F(ProtoValueWrapTest, ProtoListForEachWithIndex) { ElementsAre(Pair(0, IntValueIs(1)), Pair(1, IntValueIs(2)))); } +class ProtoValueUnwrapTest : public ProtoValueTest {}; + +TEST_F(ProtoValueUnwrapTest, SameDescriptor) { + TestAllTypes src = ParseTextOrDie( + R"pb(single_int32: 1 single_int64: 2 single_string: "hello")pb"); + ASSERT_OK_AND_ASSIGN( + auto value, + ProtoMessageToValue(src, descriptor_pool(), message_factory(), arena())); + TestAllTypes dest; + ASSERT_THAT(ProtoMessageFromValue(value, dest), IsOk()); + EXPECT_EQ(dest.single_int32(), 1); + EXPECT_EQ(dest.single_int64(), 2); + EXPECT_EQ(dest.single_string(), "hello"); +} + +TEST_F(ProtoValueUnwrapTest, DifferentDescriptorSameFullName) { + auto* dynamic_message = + DynamicParseTextProto(R"pb(single_int32: 42 + single_string: "dynamic")pb"); + Value value = Value::WrapMessage(dynamic_message, descriptor_pool(), + message_factory(), arena()); + TestAllTypes dest; + ASSERT_THAT(ProtoMessageFromValue(value, dest), IsOk()); + EXPECT_EQ(dest.single_int32(), 42); + EXPECT_EQ(dest.single_string(), "dynamic"); +} + +TEST_F(ProtoValueUnwrapTest, TypeMismatch) { + TestAllTypes::NestedMessage nested; + nested.set_bb(100); + Value value = Value::WrapMessage(&nested, descriptor_pool(), + message_factory(), arena()); + TestAllTypes dest; + EXPECT_THAT(ProtoMessageFromValue(value, dest), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST_F(ProtoValueUnwrapTest, NonMessageValue) { + TestAllTypes dest; + EXPECT_THAT(ProtoMessageFromValue(IntValue(42), dest), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(ProtoMessageFromValue(StringValue("not a message"), dest), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(ProtoMessageFromValue(NullValue(), dest), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +const google::protobuf::DescriptorPool& +GetTestExternalExtensionsDescriptorPoolUnderlay() { + static const google::protobuf::DescriptorPool* pool = []() { + auto pool = std::make_unique( + google::protobuf::DescriptorPool::generated_pool()); + google::protobuf::LinkMessageReflection(); + ABSL_CHECK( + pool->BuildFile(test::GetTestExternalExtensionsFileDescriptor())); + return pool.release(); + }(); + return *pool; +} + +std::unique_ptr MakeTestExtendedMessage( + const google::protobuf::DescriptorPool& pool, google::protobuf::MessageFactory& factory) { + const google::protobuf::Descriptor* desc = + pool.FindMessageTypeByName("cel.expr.conformance.proto2.TestAllTypes"); + ABSL_CHECK(desc != nullptr); + const google::protobuf::FieldDescriptor* ext_msg_field = + pool.FindExtensionByPrintableName( + desc, "cel.cpp.testutil.test_external_extensions_message"); + ABSL_CHECK(ext_msg_field != nullptr); + const google::protobuf::Message* prototype = factory.GetPrototype(desc); + ABSL_CHECK(prototype != nullptr); + std::unique_ptr dynamic_message(prototype->New()); + + const auto* reflection = dynamic_message->GetReflection(); + reflection->SetInt32(dynamic_message.get(), + desc->FindFieldByName("single_int32"), 42); + reflection->SetString(dynamic_message.get(), + desc->FindFieldByName("single_string"), "dynamic"); + google::protobuf::Message* ext = reflection->MutableMessage(dynamic_message.get(), + ext_msg_field, &factory); + ext->GetReflection()->SetString( + ext, ext->GetDescriptor()->FindFieldByName("string_field"), + "external_string"); + return dynamic_message; +} + +TEST_F(ProtoValueUnwrapTest, DynamicMessageFromUnderlayDescriptorPool) { + const auto& pool = GetTestExternalExtensionsDescriptorPoolUnderlay(); + TestAllTypes dest; + { + google::protobuf::DynamicMessageFactory factory(&pool); + factory.SetDelegateToGeneratedFactory(false); + std::unique_ptr dynamic_message = + MakeTestExtendedMessage(pool, factory); + + Value value = + Value::WrapMessage(dynamic_message.get(), &pool, &factory, arena()); + ASSERT_THAT(ProtoMessageFromValue(value, dest), IsOk()); + } + EXPECT_EQ(dest.single_int32(), 42); + EXPECT_EQ(dest.single_string(), "dynamic"); + EXPECT_EQ(dest.unknown_fields().field_count(), 1); +} + +using ProtoValueUnwrapTestDeathTest = ProtoValueUnwrapTest; + +TEST_F(ProtoValueUnwrapTestDeathTest, + DynamicMessageFromUnderlayDescriptorPoolDelegateToGeneratedFactory) { +#ifndef ADDRESS_SANITIZER + GTEST_SKIP() << "Test requires ASAN enabled"; +#else + EXPECT_DEATH_IF_SUPPORTED( + { + const auto& pool = GetTestExternalExtensionsDescriptorPoolUnderlay(); + auto dest = std::make_unique(); + { + // The CEL runtime should not create a factory like this. + // To get this to work, the caller would need to manage the + // message factory, provide it to the runtime, and guarantee that it + // outlives the CEL value or any derived messages. + google::protobuf::DynamicMessageFactory factory(&pool); + factory.SetDelegateToGeneratedFactory(true); + + std::unique_ptr dynamic_message = + MakeTestExtendedMessage(pool, factory); + + Value value = Value::WrapMessage(dynamic_message.get(), &pool, + &factory, arena()); + ASSERT_THAT(ProtoMessageFromValue(value, *dest), IsOk()); + } + + EXPECT_EQ(dest->single_int32(), 42); + EXPECT_EQ(dest->single_string(), "dynamic"); + EXPECT_TRUE(dest->GetReflection()->HasField( + *dest, pool.FindExtensionByName( + "cel.cpp.testutil.test_external_extensions_message"))); + dest.reset(); + }, + "AddressSanitizer: heap-use-after-free"); +#endif +} + } // namespace } // namespace cel::extensions diff --git a/internal/well_known_types.cc b/internal/well_known_types.cc index 175f978d4..5232bb6ad 100644 --- a/internal/well_known_types.cc +++ b/internal/well_known_types.cc @@ -1980,6 +1980,12 @@ absl::StatusOr> AdaptAny( } BytesValue value = reflection.GetValue(*to_unwrap, value_scratch); Unique unpacked = WrapUnique(prototype->New(arena), arena); + // TODO(b/557267722): Extensions that are not included in the same + // descriptor pool as the resolved descriptor will be treated as unknown + // fields. Extending messages like this should be exceedingly rare and is + // hard to support safely. + // + // See CodedInputStream::SetExtensionRegistry. const bool ok = absl::visit(absl::Overload( [&](absl::string_view string) -> bool { return unpacked->ParseFromString(string); diff --git a/runtime/runtime.h b/runtime/runtime.h index 2db39b0e3..8c76236dd 100644 --- a/runtime/runtime.h +++ b/runtime/runtime.h @@ -76,11 +76,20 @@ class Program { // Activation manages instances of variables available in the cel expression's // environment. // - // The arena will be used to as necessary to allocate values and must outlive - // the returned value, as must this program. + // Notes on lifetimes: // - // For consistency, users should use the same arena to create values - // in the activation and for Program evaluation. + // The provided arena will be used as necessary to allocate complex values + // and must outlive any returned value. Values created by the program may + // depend on internal state in the runtime. In particular protobuf messages + // may depend on the descriptor pool and message factory managed by the + // runtime or program. + // + // Programs implicitly keep shared state in the runtime object alive so it + // is sufficient to ensure that any cel::Value result is destroyed before the + // cel::Program that created it. + // + // For consistency, users should use the same arena to create values placed in + // the activation for calls to Program::Evaluate. absl::StatusOr Evaluate( google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND, const ActivationInterface& activation, diff --git a/runtime/runtime_builder_factory.h b/runtime/runtime_builder_factory.h index 0cb35d62a..2efe88465 100644 --- a/runtime/runtime_builder_factory.h +++ b/runtime/runtime_builder_factory.h @@ -48,8 +48,14 @@ namespace cel { // - google.protobuf.Duration // - google.protobuf.Timestamp // -// This is provided for environments that only use a subset of the CEL standard -// builtins. Most users should prefer CreateStandardRuntimeBuilder. +// Warning: It is best to use a hermetic descriptor pool (i.e. not created with +// the "underlay" constructor). There is not an efficient way to consistently +// detect whether a given message definitely has no dependency on an overlaid +// descriptor pool and associated message factory. This can hide use-after-free +// type bugs. These issues are mitigated by using a hermetic descriptor pool. +// +// This function is provided for environments that only use a subset of the CEL +// standard builtins. Most users should prefer CreateStandardRuntimeBuilder. // // Callers must register appropriate builtins. absl::StatusOr CreateRuntimeBuilder( diff --git a/testutil/BUILD b/testutil/BUILD index 3c1832652..a8bc214c9 100644 --- a/testutil/BUILD +++ b/testutil/BUILD @@ -15,6 +15,8 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_test.bzl", "cc_test") +load("//bazel:cel_cc_embed.bzl", "cel_cc_embed") +load("//bazel:cel_proto_transitive_descriptor_set.bzl", "cel_proto_transitive_descriptor_set") package(default_visibility = ["//visibility:public"]) @@ -79,3 +81,47 @@ proto_library( name = "test_json_names_proto", srcs = ["test_json_names.proto"], ) + +proto_library( + name = "test_external_extensions_proto", + srcs = ["test_external_extensions.proto"], + deps = ["@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_proto"], +) + +cel_proto_transitive_descriptor_set( + name = "test_external_extensions_transitive_descriptor_set", + testonly = True, + deps = [ + ":test_external_extensions_proto", + ], +) + +cel_cc_embed( + name = "test_external_extensions_descriptor_set_embed", + testonly = True, + src = ":test_external_extensions_transitive_descriptor_set", +) + +cc_library( + name = "test_external_extensions_descriptor_set", + testonly = True, + srcs = ["test_external_extensions_descriptor_set.cc"], + hdrs = ["test_external_extensions_descriptor_set.h"], + textual_hdrs = [":test_external_extensions_descriptor_set_embed"], + deps = [ + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/log:absl_check", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "test_external_extensions_descriptor_set_test", + srcs = ["test_external_extensions_descriptor_set_test.cc"], + deps = [ + ":test_external_extensions_descriptor_set", + "//internal:testing", + "@com_google_protobuf//:protobuf", + ], +) diff --git a/testutil/test_external_extensions.proto b/testutil/test_external_extensions.proto new file mode 100644 index 000000000..b78827b67 --- /dev/null +++ b/testutil/test_external_extensions.proto @@ -0,0 +1,42 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Tests for external extensions added in an overlay. + +edition = "2024"; + +package cel.cpp.testutil; + +import "cel/expr/conformance/proto2/test_all_types.proto"; + +// Test message for external extensions. +message TestExternalExtensions { + string string_field = 1; + int64 int64_field = 2; +} + +extend cel.expr.conformance.proto2.TestAllTypes { + TestExternalExtensions test_external_extensions_message = 2001; + string test_external_extensions_string = 2002; + int64 test_external_extensions_int64 = 2003; +} + +// Message scoped extension. +message ScopedTestExternalExtensions { + extend cel.expr.conformance.proto2.TestAllTypes { + TestExternalExtensions scoped_ext_message = 3001; + string scoped_ext_string = 3002; + int64 scoped_ext_int64 = 3003; + } +} diff --git a/testutil/test_external_extensions_descriptor_set.cc b/testutil/test_external_extensions_descriptor_set.cc new file mode 100644 index 000000000..d18633a03 --- /dev/null +++ b/testutil/test_external_extensions_descriptor_set.cc @@ -0,0 +1,62 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "testutil/test_external_extensions_descriptor_set.h" + +#include +#include + +#include "google/protobuf/descriptor.pb.h" +#include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" +#include "absl/log/absl_check.h" + +namespace cel::test { + +namespace { + +ABSL_CONST_INIT const uint8_t kTestExternalExtensionsDescriptorSet[] = { +#include "testutil/test_external_extensions_descriptor_set_embed.inc" +}; + +} // namespace + +const google::protobuf::FileDescriptorSet& GetTestExternalExtensionsFileDescriptorSet() { + static const absl::NoDestructor + file_descriptor_set([]() { + google::protobuf::FileDescriptorSet file_desc_set; + ABSL_CHECK(file_desc_set.ParseFromArray( // Crash OK + kTestExternalExtensionsDescriptorSet, + std::size(kTestExternalExtensionsDescriptorSet))); + return file_desc_set; + }()); + return *file_descriptor_set; +} + +const google::protobuf::FileDescriptorProto& GetTestExternalExtensionsFileDescriptor() { + static const google::protobuf::FileDescriptorProto* const file_desc = []() { + const auto& file_desc_set = GetTestExternalExtensionsFileDescriptorSet(); + for (const auto& file_desc : file_desc_set.file()) { + if (file_desc.name() == + "testutil/test_external_extensions.proto") { + return &file_desc; + } + } + ABSL_UNREACHABLE(); + }(); + return *file_desc; +} + +} // namespace cel::test diff --git a/testutil/test_external_extensions_descriptor_set.h b/testutil/test_external_extensions_descriptor_set.h new file mode 100644 index 000000000..a5302e1bb --- /dev/null +++ b/testutil/test_external_extensions_descriptor_set.h @@ -0,0 +1,28 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_EXTERNAL_EXTENSIONS_DESCRIPTOR_SET_H_ +#define THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_EXTERNAL_EXTENSIONS_DESCRIPTOR_SET_H_ + +#include "google/protobuf/descriptor.pb.h" + +namespace cel::test { + +const google::protobuf::FileDescriptorSet& GetTestExternalExtensionsFileDescriptorSet(); + +const google::protobuf::FileDescriptorProto& GetTestExternalExtensionsFileDescriptor(); + +} // namespace cel::test + +#endif // THIRD_PARTY_CEL_CPP_TESTUTIL_TEST_EXTERNAL_EXTENSIONS_DESCRIPTOR_SET_H_ diff --git a/testutil/test_external_extensions_descriptor_set_test.cc b/testutil/test_external_extensions_descriptor_set_test.cc new file mode 100644 index 000000000..de4b61cba --- /dev/null +++ b/testutil/test_external_extensions_descriptor_set_test.cc @@ -0,0 +1,56 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "testutil/test_external_extensions_descriptor_set.h" + +#include "internal/testing.h" +#include "google/protobuf/descriptor.h" + +namespace cel::test { +namespace { + +using ::testing::NotNull; + +TEST(GetTestExternalExtensionsFileDescriptorSet, BuildsPool) { + const auto& fds = GetTestExternalExtensionsFileDescriptorSet(); + EXPECT_GT(fds.file_size(), 0); + + google::protobuf::DescriptorPool pool; + for (const auto& file : fds.file()) { + EXPECT_THAT(pool.BuildFile(file), NotNull()); + } + + EXPECT_THAT( + pool.FindMessageTypeByName("cel.cpp.testutil.TestExternalExtensions"), + NotNull()); + EXPECT_THAT(pool.FindExtensionByName( + "cel.cpp.testutil.test_external_extensions_message"), + NotNull()); + EXPECT_THAT( + pool.FindExtensionByName( + "cel.cpp.testutil.ScopedTestExternalExtensions.scoped_ext_string"), + NotNull()); +} + +TEST(GetTestExternalExtensionsFileDescriptor, CorrectFileDescriptor) { + const auto& fd = GetTestExternalExtensionsFileDescriptor(); + EXPECT_EQ(fd.name(), + "testutil/test_external_extensions.proto"); + EXPECT_EQ(fd.package(), "cel.cpp.testutil"); + EXPECT_GT(fd.message_type_size(), 0); + EXPECT_GT(fd.extension_size(), 0); +} + +} // namespace +} // namespace cel::test From 705fcdb2143251b4622801274368d91320a47560 Mon Sep 17 00:00:00 2001 From: Sean Huh Date: Thu, 10 Sep 2026 14:26:15 -0700 Subject: [PATCH 214/350] Fix type unification for type parameters PiperOrigin-RevId: 979398265 --- checker/internal/type_inference_context.cc | 40 ++- .../internal/type_inference_context_test.cc | 262 ++++++++++++++++++ checker/optional_test.cc | 34 +++ checker/standard_library.cc | 6 +- 4 files changed, 332 insertions(+), 10 deletions(-) diff --git a/checker/internal/type_inference_context.cc b/checker/internal/type_inference_context.cc index 4f738b804..4f0cb2d5c 100644 --- a/checker/internal/type_inference_context.cc +++ b/checker/internal/type_inference_context.cc @@ -153,6 +153,19 @@ std::optional WrapperToPrimitive(const Type& t) { } } +// Tests whether the type contains any type params directly or transitively. +bool HasTypeParam(const Type& type) { + if (type.kind() == TypeKind::kTypeParam) { + return true; + } + for (const auto& param : type.GetParameters()) { + if (HasTypeParam(param)) { + return true; + } + } + return false; +} + } // namespace Type TypeInferenceContext::InstantiateTypeParams(const Type& type) { @@ -252,6 +265,26 @@ bool TypeInferenceContext::IsAssignableInternal( Type to_subs = Substitute(to, prospective_substitutions); Type from_subs = Substitute(from, prospective_substitutions); + if (from_subs.kind() == TypeKind::kType && + to_subs.kind() == TypeKind::kType) { + Type from_inner = from_subs.AsType()->GetType(); + Type to_inner = to_subs.AsType()->GetType(); + // If either type contains a type parameter (e.g., type(T) in foo(data, + // type(T)) -> T), delegate to inner type unification to bind or validate + // type parameter substitutions. Returns true if the inner types + // structurally match, unify with an unbound type param, or conform to an + // existing binding in 'prospective_substitutions'. Returns false on + // structural/kind mismatches (e.g., int vs list(T)), occurs-check cycles, + // or conflicting type param bindings. + if (HasTypeParam(from_inner) || HasTypeParam(to_inner)) { + return IsAssignableInternal(from_inner, to_inner, + prospective_substitutions); + } + // Concrete types are coassignable in CEL (e.g., type(1) == type("a"), + // type([1]) == list). + return true; + } + // Types always assignable to themselves. // Remainder is checking for assignability across different types. if (to_subs == from_subs) { @@ -313,13 +346,6 @@ bool TypeInferenceContext::IsAssignableInternal( } } - if (from_subs.kind() == TypeKind::kType && - to_subs.kind() == TypeKind::kType) { - // Types are always assignable to themselves (even if differently - // parameterized). - return true; - } - if (to_subs.kind() == TypeKind::kEnum && from_subs.kind() == TypeKind::kInt) { return true; } diff --git a/checker/internal/type_inference_context_test.cc b/checker/internal/type_inference_context_test.cc index 458d08ff1..becf0a979 100644 --- a/checker/internal/type_inference_context_test.cc +++ b/checker/internal/type_inference_context_test.cc @@ -846,5 +846,267 @@ TEST(TypeInferenceContextTest, AssignabilityContextReset) { IsTypeKind(TypeKind::kDouble)); } +TEST(TypeInferenceContextTest, + TypeTypeAssignability_ConcreteTypes_Coassignable) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type int_type = TypeType(&arena, IntType()); + Type string_type = TypeType(&arena, StringType()); + + EXPECT_TRUE(context.IsAssignable(int_type, string_type)); + EXPECT_TRUE(context.IsAssignable(string_type, int_type)); +} + +TEST(TypeInferenceContextTest, TypeTypeAssignability_MapContainerErasure) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type map_int_uint = TypeType(&arena, MapType(&arena, IntType(), UintType())); + Type map_dyn_dyn = TypeType(&arena, MapType(&arena, DynType(), DynType())); + + EXPECT_TRUE(context.IsAssignable(map_int_uint, map_dyn_dyn)); +} + +TEST(TypeInferenceContextTest, TypeTypeAssignability_ListContainerErasure) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type list_int = TypeType(&arena, ListType(&arena, IntType())); + Type list_dyn = TypeType(&arena, ListType(&arena, DynType())); + + EXPECT_TRUE(context.IsAssignable(list_int, list_dyn)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_TypeParamTarget_BindsConcreteType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type type_param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type from_type = TypeType(&arena, IntType()); + + EXPECT_TRUE(context.IsAssignable(from_type, type_param_t)); + + Type resolved_type = context.FinalizeType(type_param_t); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + EXPECT_THAT(resolved_type.AsType()->GetParameters(), + ElementsAre(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_TypeParamSource_BindsConcreteType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type type_param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type to_type = TypeType(&arena, IntType()); + + EXPECT_TRUE(context.IsAssignable(type_param_t, to_type)); + + Type resolved_type = context.FinalizeType(type_param_t); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + EXPECT_THAT(resolved_type.AsType()->GetParameters(), + ElementsAre(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, TypeTypeAssignability_NestedTypeParam_Unifies) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type type_param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type type_param_r = context.InstantiateTypeParams( + TypeType(&arena, TypeType(&arena, TypeParamType("R")))); + + EXPECT_TRUE(context.IsAssignable(type_param_t, type_param_r)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_DeeplyNestedTypeParam_BindsConcreteType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = TypeType(&arena, TypeType(&arena, IntType())); + Type to_type = context.InstantiateTypeParams( + TypeType(&arena, TypeType(&arena, TypeParamType("T")))); + + EXPECT_TRUE(context.IsAssignable(from_type, to_type)); + + Type resolved_type = context.FinalizeType(to_type); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + Type inner_type = resolved_type.AsType()->GetType(); + ASSERT_THAT(inner_type, IsTypeKind(TypeKind::kType)); + EXPECT_THAT(inner_type.AsType()->GetParameters(), + ElementsAre(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_CompositeListTypeParam_BindsConcreteType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = TypeType(&arena, ListType(&arena, IntType())); + Type to_type = context.InstantiateTypeParams( + TypeType(&arena, ListType(&arena, TypeParamType("T")))); + + EXPECT_TRUE(context.IsAssignable(from_type, to_type)); + + Type resolved_type = context.FinalizeType(to_type); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + Type inner_type = resolved_type.AsType()->GetType(); + ASSERT_THAT(inner_type, IsTypeKind(TypeKind::kList)); + EXPECT_THAT(inner_type.AsList()->GetElement(), IsTypeKind(TypeKind::kInt)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_CompositeMapTypeParam_BindsConcreteTypes) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = TypeType(&arena, MapType(&arena, StringType(), IntType())); + Type to_type = context.InstantiateTypeParams(TypeType( + &arena, MapType(&arena, TypeParamType("K"), TypeParamType("V")))); + + EXPECT_TRUE(context.IsAssignable(from_type, to_type)); + + Type resolved_type = context.FinalizeType(to_type); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + Type inner_type = resolved_type.AsType()->GetType(); + ASSERT_THAT(inner_type, IsTypeKind(TypeKind::kMap)); + EXPECT_THAT(inner_type.AsMap()->GetKey(), IsTypeKind(TypeKind::kString)); + EXPECT_THAT(inner_type.AsMap()->GetValue(), IsTypeKind(TypeKind::kInt)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OptionalTypeParam_Unifies) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = TypeType(&arena, OptionalType(&arena, IntType())); + Type to_type = context.InstantiateTypeParams( + TypeType(&arena, OptionalType(&arena, TypeParamType("T")))); + + EXPECT_TRUE(context.IsAssignable(from_type, to_type)); + + Type resolved_type = context.FinalizeType(to_type); + ASSERT_THAT(resolved_type, IsTypeKind(TypeKind::kType)); + Type inner_type = resolved_type.AsType()->GetType(); + ASSERT_THAT(inner_type, IsTypeKind(TypeKind::kOpaque)); + EXPECT_THAT(inner_type.AsOpaque()->GetParameters(), + ElementsAre(IsTypeKind(TypeKind::kInt))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_IncompatibleTypeParams_ReturnsFalse) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type from_type = context.InstantiateTypeParams( + TypeType(&arena, ListType(&arena, TypeParamType("T")))); + Type to_type = TypeType(&arena, IntType()); + + EXPECT_FALSE(context.IsAssignable(from_type, to_type)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_ConflictingBoundTypeParam_ReturnsFalse) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + EXPECT_TRUE(context.IsAssignable(TypeType(&arena, StringType()), param_t)); + EXPECT_FALSE(context.IsAssignable(param_t, TypeType(&arena, IntType()))); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OccursCheck_FailsOnSelfReference) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type to_type = TypeType(&arena, param_t); + + EXPECT_FALSE(context.IsAssignable(param_t, to_type)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OccursCheck_FailsOnTransitiveCycle) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type param_r = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("R"))); + + EXPECT_TRUE(context.IsAssignable(param_t, TypeType(&arena, param_r))); + EXPECT_FALSE(context.IsAssignable(param_r, TypeType(&arena, param_t))); +} + +TEST(TypeInferenceContextTest, + TypeTypeOverloadResolution_TypeParamInTypeType_ResolvesReturnTypeInt) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl decl, + MakeFunctionDecl("cast", + MakeOverloadDecl("cast_t", TypeParamType("T"), DynType(), + TypeType(&arena, TypeParamType("T"))))); + + std::optional resolution = + context.ResolveOverload(decl, {StringType(), TypeType(&arena, IntType())}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_THAT(context.FinalizeType(resolution->result_type), + IsTypeKind(TypeKind::kInt)); +} + +TEST(TypeInferenceContextTest, + TypeTypeOverloadResolution_TypeParamInTypeType_ResolvesReturnTypeString) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl decl, + MakeFunctionDecl("cast", + MakeOverloadDecl("cast_t", TypeParamType("T"), DynType(), + TypeType(&arena, TypeParamType("T"))))); + + std::optional resolution = + context.ResolveOverload(decl, {IntType(), TypeType(&arena, StringType())}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_THAT(context.FinalizeType(resolution->result_type), + IsTypeKind(TypeKind::kString)); +} + +TEST(TypeInferenceContextTest, + TypeTypeOverloadResolution_TypeParamInComposite_ResolvesReturnType) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + ASSERT_OK_AND_ASSIGN( + FunctionDecl decl, + MakeFunctionDecl( + "first_elem_type", + MakeOverloadDecl( + "first_elem_type_overload", TypeParamType("T"), DynType(), + TypeType(&arena, ListType(&arena, TypeParamType("T")))))); + + std::optional resolution = + context.ResolveOverload( + decl, {StringType(), TypeType(&arena, ListType(&arena, IntType()))}, + false); + ASSERT_TRUE(resolution.has_value()); + EXPECT_THAT(context.FinalizeType(resolution->result_type), + IsTypeKind(TypeKind::kInt)); +} + } // namespace } // namespace cel::checker_internal diff --git a/checker/optional_test.cc b/checker/optional_test.cc index 87c14f0cd..aa8a8712c 100644 --- a/checker/optional_test.cc +++ b/checker/optional_test.cc @@ -335,5 +335,39 @@ INSTANTIATE_TEST_SUITE_P( "== null", _, "no matching overload for '_==_'"})); +class OptionalListTypePermutationsTest + : public testing::TestWithParam {}; + +TEST_P(OptionalListTypePermutationsTest, ResolvesToListDyn) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr builder, + CreateTypeCheckerBuilder(GetSharedTestingDescriptorPool())); + ASSERT_THAT(builder->AddLibrary(StandardCheckerLibrary()), IsOk()); + ASSERT_THAT(builder->AddLibrary(OptionalCheckerLibrary()), IsOk()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr checker, + std::move(*builder).Build()); + + const std::string& expr = GetParam(); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst(expr)); + ASSERT_OK_AND_ASSIGN(auto result, checker->Check(std::move(ast))); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetTypeOrDyn(checked_ast->root_expr().id()), + TypeSpec(ListTypeSpec(std::make_unique(DynTypeSpec())))); + ASSERT_EQ(checked_ast->root_expr().list_expr().elements().size(), 3); + for (const auto& elem : checked_ast->root_expr().list_expr().elements()) { + EXPECT_TRUE(checked_ast->GetTypeOrDyn(elem.expr().id()).has_type()); + } +} + +INSTANTIATE_TEST_SUITE_P( + OptionalTests, OptionalListTypePermutationsTest, + ::testing::Values("[type([]), int, type(optional.none())]", + "[type([]), type(optional.none()), int]", + "[int, type([]), type(optional.none())]", + "[int, type(optional.none()), type([])]", + "[type(optional.none()), type([]), int]", + "[type(optional.none()), int, type([])]")); + } // namespace } // namespace cel diff --git a/checker/standard_library.cc b/checker/standard_library.cc index 744a171ef..0162db003 100644 --- a/checker/standard_library.cc +++ b/checker/standard_library.cc @@ -112,13 +112,13 @@ Type TypeDynType() { Type TypeListType() { static absl::NoDestructor kInstance( - TypeType(BuiltinsArena(), ListOfA())); + TypeType(BuiltinsArena(), ListType(BuiltinsArena(), DynType()))); return *kInstance; } Type TypeMapType() { - static absl::NoDestructor kInstance( - TypeType(BuiltinsArena(), MapOfAB())); + static absl::NoDestructor kInstance(TypeType( + BuiltinsArena(), MapType(BuiltinsArena(), DynType(), DynType()))); return *kInstance; } From 689dba4c89d994342b1d0d870bff39dfbd7acb35 Mon Sep 17 00:00:00 2001 From: Sean Huh Date: Fri, 11 Sep 2026 13:54:54 -0700 Subject: [PATCH 215/350] Fix type checking around comprehensions with empty iteration range PiperOrigin-RevId: 980003369 --- checker/internal/type_checker_impl.cc | 10 +++- checker/internal/type_checker_impl_test.cc | 70 ++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index bca187417..0e71e864a 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -838,7 +838,8 @@ void ResolveVisitor::PostVisitComprehensionSubexpression( GetDeducedType(&comprehension.accu_init()))); break; case ComprehensionArg::ITER_RANGE: { - Type range_type = GetDeducedType(&comprehension.iter_range()); + Type range_type = inference_context_->FullySubstitute( + GetDeducedType(&comprehension.iter_range()), /*free_to_dyn=*/false); Type iter_type = DynType(); // iter_var for non comprehensions v2. Type iter_type1 = DynType(); // iter_var for comprehensions v2. Type iter_type2 = DynType(); // iter_var2 for comprehensions v2. @@ -851,7 +852,14 @@ void ResolveVisitor::PostVisitComprehensionSubexpression( iter_type = iter_type1 = range_type.GetMap().key(); iter_type2 = range_type.GetMap().value(); break; + case TypeKind::kTypeParam: + // Set the range type to DYN to prevent assignment to a potentially + // incorrect type at a later point in type-checking. The IsAssignable + // call will update the type substitutions for the type param. + inference_context_->IsAssignable(DynType(), range_type); + break; case TypeKind::kDyn: + case TypeKind::kError: break; default: ReportIssue(TypeCheckIssue::CreateError( diff --git a/checker/internal/type_checker_impl_test.cc b/checker/internal/type_checker_impl_test.cc index 61ef7d55b..76e1bfa13 100644 --- a/checker/internal/type_checker_impl_test.cc +++ b/checker/internal/type_checker_impl_test.cc @@ -1287,6 +1287,76 @@ TEST(TypeCheckerImplTest, ComprehensionDynRange) { EXPECT_THAT(result.GetIssues(), IsEmpty()); } +TEST(TypeCheckerImplTest, EmptyListRangeNestedListComprehension) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("[].map(x, x.map(y, y))")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetReturnType(), + AstType(ListTypeSpec(std::make_unique( + ListTypeSpec(std::make_unique(DynTypeSpec())))))); +} + +TEST(TypeCheckerImplTest, EmptyMapRangeNestedComprehension) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(auto ast, MakeTestParsedAst("{}.map(k, k.map(y, y))")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetReturnType(), + AstType(ListTypeSpec(std::make_unique( + ListTypeSpec(std::make_unique(DynTypeSpec())))))); +} + +TEST(TypeCheckerImplTest, EmptyRangeNestedComprehensionPredicate) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN(auto ast, + MakeTestParsedAst("[].all(x, x.all(y, y == 1))")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetReturnType(), AstType(PrimitiveType::kBool)); +} + +TEST(TypeCheckerImplTest, EmptyRangeNestedComprehensionBeforeConstraint) { + TypeCheckEnv env(GetSharedTestingDescriptorPool()); + google::protobuf::Arena arena; + ASSERT_THAT(RegisterMinimalBuiltins(&arena, env), IsOk()); + + TypeCheckerImpl impl(std::move(env)); + ASSERT_OK_AND_ASSIGN( + auto ast, MakeTestParsedAst("[].all(x, x.all(y, y == 1) && x == 1)")); + ASSERT_OK_AND_ASSIGN(ValidationResult result, impl.Check(std::move(ast))); + + EXPECT_TRUE(result.IsValid()) << result.FormatError(); + EXPECT_THAT(result.GetIssues(), IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto checked_ast, result.ReleaseAst()); + EXPECT_EQ(checked_ast->GetReturnType(), AstType(PrimitiveType::kBool)); +} + TEST(TypeCheckerImplTest, BasicOvlResolution) { TypeCheckEnv env(GetSharedTestingDescriptorPool()); google::protobuf::Arena arena; From e0daceaba26b85e349bc208e04677b1bfe14dbad Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Fri, 11 Sep 2026 18:51:07 -0700 Subject: [PATCH 216/350] No public description PiperOrigin-RevId: 980136290 --- checker/internal/type_inference_context.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/checker/internal/type_inference_context.cc b/checker/internal/type_inference_context.cc index 4f0cb2d5c..41f3a8b48 100644 --- a/checker/internal/type_inference_context.cc +++ b/checker/internal/type_inference_context.cc @@ -199,7 +199,8 @@ Type TypeInferenceContext::InstantiateTypeParams( case TypeKind::kUintWrapper: return type; case TypeKind::kTypeParam: { - absl::string_view name = type.AsTypeParam()->name(); + TypeParamType type_param = type.GetTypeParam(); + absl::string_view name = type_param.name(); if (IsTypeVar(name)) { // Already instantiated (e.g. list comprehension variable). return type; @@ -494,7 +495,8 @@ bool TypeInferenceContext::OccursWithin( // T3 -> T2 -> null_type; Type substitution = type; while (substitution.kind() == TypeKind::kTypeParam) { - absl::string_view param_name = substitution.AsTypeParam()->name(); + TypeParamType type_param = substitution.GetTypeParam(); + absl::string_view param_name = type_param.name(); if (param_name == var_name) { return true; } @@ -534,7 +536,8 @@ bool TypeInferenceContext::IsAssignableWithConstraints( } if (to.kind() == TypeKind::kTypeParam) { - absl::string_view name = to.AsTypeParam()->name(); + TypeParamType to_param = to.GetTypeParam(); + absl::string_view name = to_param.name(); if (!OccursWithin(name, from, prospective_substitutions)) { prospective_substitutions[name] = from; return true; @@ -542,7 +545,8 @@ bool TypeInferenceContext::IsAssignableWithConstraints( } if (from.kind() == TypeKind::kTypeParam) { - absl::string_view name = from.AsTypeParam()->name(); + TypeParamType from_param = from.GetTypeParam(); + absl::string_view name = from_param.name(); if (!OccursWithin(name, to, prospective_substitutions)) { prospective_substitutions[name] = to; return true; @@ -663,11 +667,12 @@ Type TypeInferenceContext::FullySubstitute(const Type& type, return MapType(arena_, key, value); } case TypeKind::kOpaque: { + OpaqueType opaque_type = type.GetOpaque(); std::vector types; - for (const auto& param : type.AsOpaque()->GetParameters()) { + for (const auto& param : opaque_type.GetParameters()) { types.push_back(FullySubstitute(param, free_to_dyn)); } - return OpaqueType(arena_, type.AsOpaque()->name(), types); + return OpaqueType(arena_, opaque_type.name(), types); } default: return type; From 0a38cb9a160fb1b2c5ceb42c22760b069611973d Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 14 Sep 2026 11:54:37 -0700 Subject: [PATCH 217/350] No public description PiperOrigin-RevId: 981274663 --- common/expr.h | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/common/expr.h b/common/expr.h index 7305c2c9f..8bf30eda1 100644 --- a/common/expr.h +++ b/common/expr.h @@ -1378,7 +1378,7 @@ inline Expr& Expr::operator=(Expr&&) = default; inline void Expr::set_kind(ExprKind kind) { kind_ = std::move(kind); } -inline ABSL_MUST_USE_RESULT ExprKind Expr::release_kind() { +[[nodiscard]] inline ExprKind Expr::release_kind() { ExprKind kind = std::move(kind_); kind_.emplace(); return kind; @@ -1388,7 +1388,7 @@ inline void Expr::set_call_expr(CallExpr call_expr) { try_emplace_kind() = std::move(call_expr); } -inline ABSL_MUST_USE_RESULT CallExpr Expr::release_call_expr() { +[[nodiscard]] inline CallExpr Expr::release_call_expr() { return release_kind(); } @@ -1499,12 +1499,12 @@ inline void ListExprElement::Clear() { optional_ = false; } -inline ABSL_MUST_USE_RESULT const Expr& ListExprElement::expr() const +[[nodiscard]] inline const Expr& ListExprElement::expr() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return has_expr() ? *expr_ : Expr::default_instance(); } -inline ABSL_MUST_USE_RESULT Expr& ListExprElement::mutable_expr() +[[nodiscard]] inline Expr& ListExprElement::mutable_expr() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (!has_expr()) { expr_ = std::make_unique(); @@ -1520,7 +1520,7 @@ inline void ListExprElement::set_expr(std::unique_ptr expr) { expr_ = std::move(expr); } -inline ABSL_MUST_USE_RESULT Expr ListExprElement::release_expr() { +[[nodiscard]] inline Expr ListExprElement::release_expr() { return release(expr_); } @@ -1570,12 +1570,12 @@ inline void StructExprField::Clear() { optional_ = false; } -inline ABSL_MUST_USE_RESULT const Expr& StructExprField::value() const +[[nodiscard]] inline const Expr& StructExprField::value() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return has_value() ? *value_ : Expr::default_instance(); } -inline ABSL_MUST_USE_RESULT Expr& StructExprField::mutable_value() +[[nodiscard]] inline Expr& StructExprField::mutable_value() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (!has_value()) { value_ = std::make_unique(); @@ -1591,7 +1591,7 @@ inline void StructExprField::set_value(std::unique_ptr value) { value_ = std::move(value); } -inline ABSL_MUST_USE_RESULT Expr StructExprField::release_value() { +[[nodiscard]] inline Expr StructExprField::release_value() { return release(value_); } @@ -1646,12 +1646,12 @@ inline void MapExprEntry::Clear() { optional_ = false; } -inline ABSL_MUST_USE_RESULT const Expr& MapExprEntry::key() const +[[nodiscard]] inline const Expr& MapExprEntry::key() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return has_key() ? *key_ : Expr::default_instance(); } -inline ABSL_MUST_USE_RESULT Expr& MapExprEntry::mutable_key() +[[nodiscard]] inline Expr& MapExprEntry::mutable_key() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (!has_key()) { key_ = std::make_unique(); @@ -1665,16 +1665,14 @@ inline void MapExprEntry::set_key(std::unique_ptr key) { key_ = std::move(key); } -inline ABSL_MUST_USE_RESULT Expr MapExprEntry::release_key() { - return release(key_); -} +[[nodiscard]] inline Expr MapExprEntry::release_key() { return release(key_); } -inline ABSL_MUST_USE_RESULT const Expr& MapExprEntry::value() const +[[nodiscard]] inline const Expr& MapExprEntry::value() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return has_value() ? *value_ : Expr::default_instance(); } -inline ABSL_MUST_USE_RESULT Expr& MapExprEntry::mutable_value() +[[nodiscard]] inline Expr& MapExprEntry::mutable_value() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (!has_value()) { value_ = std::make_unique(); @@ -1690,7 +1688,7 @@ inline void MapExprEntry::set_value(std::unique_ptr value) { value_ = std::move(value); } -inline ABSL_MUST_USE_RESULT Expr MapExprEntry::release_value() { +[[nodiscard]] inline Expr MapExprEntry::release_value() { return release(value_); } From f7b0b752bb2186cd3fc49ea3b96dfbf4d8a3516b Mon Sep 17 00:00:00 2001 From: Sean Huh Date: Tue, 15 Sep 2026 11:50:41 -0700 Subject: [PATCH 218/350] Add test cases around map type parameter propagation PiperOrigin-RevId: 981946267 --- .../internal/type_inference_context_test.cc | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/checker/internal/type_inference_context_test.cc b/checker/internal/type_inference_context_test.cc index becf0a979..00f409253 100644 --- a/checker/internal/type_inference_context_test.cc +++ b/checker/internal/type_inference_context_test.cc @@ -1048,6 +1048,32 @@ TEST(TypeInferenceContextTest, EXPECT_FALSE(context.IsAssignable(param_r, TypeType(&arena, param_t))); } +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OccursCheck_MapTypeParamToTypeParam_ReturnsFalse) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type t_var = param_t.AsType()->GetType(); + Type map_type = TypeType(&arena, MapType(&arena, StringType(), t_var)); + + EXPECT_FALSE(context.IsAssignable(map_type, param_t)); +} + +TEST(TypeInferenceContextTest, + TypeTypeAssignability_OccursCheck_TypeParamToMapTypeParam_ReturnsFalse) { + google::protobuf::Arena arena; + TypeInferenceContext context(&arena); + + Type param_t = + context.InstantiateTypeParams(TypeType(&arena, TypeParamType("T"))); + Type t_var = param_t.AsType()->GetType(); + Type map_type = TypeType(&arena, MapType(&arena, StringType(), t_var)); + + EXPECT_FALSE(context.IsAssignable(param_t, map_type)); +} + TEST(TypeInferenceContextTest, TypeTypeOverloadResolution_TypeParamInTypeType_ResolvesReturnTypeInt) { google::protobuf::Arena arena; From 754d1e82a57da37671cbacaaa39a5d11590368c3 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 15 Sep 2026 15:13:52 -0700 Subject: [PATCH 219/350] Simplify SavePosition/RestorePosition functions PiperOrigin-RevId: 982068463 --- parser/internal/lexer.cc | 2 -- parser/internal/lexer.h | 28 ++++++--------------------- parser/internal/lexer_test.cc | 3 ++- parser/internal/pratt_parser_worker.h | 2 +- 4 files changed, 9 insertions(+), 26 deletions(-) diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc index db08b644f..897a63399 100644 --- a/parser/internal/lexer.cc +++ b/parser/internal/lexer.cc @@ -163,8 +163,6 @@ std::string_view TokenTypeToString(TokenType type) { Token Lexer::Lex() { int32_t start = GetPosition(); if (ABSL_PREDICT_FALSE(position_ >= content_.size())) { - at_end_ = true; - done_ = true; return MakeToken(TokenType::kEnd, start, start); } char32_t c = content_.at(position_); diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h index c35dc08e2..502be627a 100644 --- a/parser/internal/lexer.h +++ b/parser/internal/lexer.h @@ -140,13 +140,6 @@ class Lexer final { std::numeric_limits::max())); } - struct Position final { - int32_t position = 0; - bool at_end = false; - bool done = false; - LexerError error; - }; - Lexer(const Lexer&) = delete; Lexer(Lexer&&) = delete; Lexer& operator=(const Lexer&) = delete; @@ -165,15 +158,13 @@ class Lexer final { [[nodiscard]] int32_t GetPosition() const { return position_; } - [[nodiscard]] Position SavePosition() const { - return Position{position_, at_end_, done_, error_}; - } + [[nodiscard]] int32_t SavePosition() const { return position_; } - void RestorePosition(const Position& position) { - position_ = position.position; - at_end_ = position.at_end; - done_ = position.done; - error_ = position.error; + void RestorePosition(int32_t position) { + ABSL_DCHECK_GE(position, 0); + ABSL_DCHECK_LE(position, static_cast(content_.size())); + position_ = position; + error_ = LexerError{}; } private: @@ -201,9 +192,6 @@ class Lexer final { } [[nodiscard]] Token MakeToken(TokenType type, int32_t start, int32_t end) { - if (ABSL_PREDICT_FALSE(at_end_)) { - AtEndTokenCreated(); - } return Token{.type = type, .start = start, .end = end}; } @@ -214,8 +202,6 @@ class Lexer final { return Token{.type = TokenType::kError, .start = start, .end = end}; } - void AtEndTokenCreated() { done_ = true; } - // Consumes characters up to and including the first occurrence of character // `c` without interpreting backslashes as escapes. Returns true if `c` was // found and consumed; false if end of input was reached. @@ -294,8 +280,6 @@ class Lexer final { cel::SourceContentView content_; int32_t position_ = 0; - bool at_end_ = false; - bool done_ = false; LexerError error_; }; diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc index ebe0011d3..9cdfbd45d 100644 --- a/parser/internal/lexer_test.cc +++ b/parser/internal/lexer_test.cc @@ -14,6 +14,7 @@ #include "parser/internal/lexer.h" +#include #include #include #include @@ -506,7 +507,7 @@ TEST(LexerPositionTest, SaveAndRestorePosition) { EXPECT_EQ(tok2.type, TokenType::kWhitespace); // Save position before '+' - Lexer::Position saved = lexer.SavePosition(); + int32_t saved = lexer.SavePosition(); Token tok3 = lexer.Lex(); EXPECT_EQ(tok3.type, TokenType::kPlus); diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index e1608a3a2..2cfe84b79 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -1222,7 +1222,7 @@ int PrattParserWorker::CountGroupingParentheses() { } // Save lexer position to restore after scanning ahead. - const Lexer::Position saved_pos = lexer_.SavePosition(); + const int32_t saved_pos = lexer_.SavePosition(); auto restore_lexer = absl::MakeCleanup( [this, saved_pos] { lexer_.RestorePosition(saved_pos); }); From 5849610484f1bdcf709f6cec8150d9028edf7fa9 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 15 Sep 2026 17:22:14 -0700 Subject: [PATCH 220/350] Fix parsing of unescaped newlines in string literals PiperOrigin-RevId: 982139232 --- parser/internal/lexer.cc | 61 ++++++------- parser/internal/lexer.h | 12 +-- parser/internal/lexer_test.cc | 30 ++++--- parser/internal/pratt_parser_test.cc | 108 +++++++++++++++++++++++ parser/parser_test.cc | 127 ++++++++++++++++++++++++++- 5 files changed, 281 insertions(+), 57 deletions(-) diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc index 897a63399..4163833be 100644 --- a/parser/internal/lexer.cc +++ b/parser/internal/lexer.cc @@ -333,17 +333,31 @@ Token Lexer::Lex() { return SetError(start, GetPosition(), "unexpected character"); } -// Consumes characters up to and including the first occurrence of character `c` -// without interpreting backslashes as escapes. -// Returns true if `c` was found and consumed; false if end of input was -// reached. -bool Lexer::ConsumeUntilAfter(char32_t c) { +// Consumes characters up to and including the first occurrence of character +// `c`. If `is_raw` is false, backslashes are interpreted as escapes. Returns +// true if `c` was found and consumed; false if end of input or an unescaped +// newline was reached. +bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { ABSL_DCHECK_NE(c, '\n'); - for (int32_t pos = position_; pos < content_.size(); ++pos) { - if (content_.at(pos) == c) { - AdvanceProcessingNewLines(pos + 1); - return true; + ABSL_DCHECK_NE(c, '\r'); + int32_t pos = position_; + bool escaped = false; + while (pos < content_.size()) { + char32_t cc = content_.at(pos); + if (cc == '\n' || cc == '\r') { + AdvanceProcessingNewLines(pos); + return false; } + if (!is_raw && cc == '\\') { + escaped = !escaped; + } else { + if (cc == c && (is_raw || !escaped)) { + AdvanceProcessingNewLines(pos + 1); + return true; + } + escaped = false; + } + ++pos; } AdvanceProcessingNewLines(content_.size()); return false; @@ -374,31 +388,6 @@ bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { return false; } -// Consumes characters up to and including the first occurrence of `c` that is -// not preceded by an odd number of backslash ('\') escape characters. Returns -// true if an unescaped `c` was found and consumed; false if reached EOF. -bool Lexer::ConsumeUntilAfterUnescaped(char32_t c) { - ABSL_DCHECK_NE(c, '\n'); - ABSL_DCHECK_NE(c, '\\'); - int32_t pos = position_; - bool escaped = false; - while (pos < content_.size()) { - char32_t cc = content_.at(pos); - if (cc == '\\') { - escaped = !escaped; - } else { - if (cc == c && !escaped) { - AdvanceProcessingNewLines(pos + 1); - return true; - } - escaped = false; - } - ++pos; - } - AdvanceProcessingNewLines(content_.size()); - return false; -} - // Consumes characters up to and including the first occurrence of substring `s` // where the first character of `s` is not preceded by an odd number of // backslashes. Returns true if an unescaped `s` was found and consumed; false @@ -562,7 +551,7 @@ TokenType Lexer::ConsumeIntegralSuffix() { Token Lexer::ConsumeQuotedIdent() { int32_t start = GetPosition(); Advance(1); - if (!ConsumeUntilAfter('`')) { + if (!ConsumeUntilAfter('`', /*is_raw=*/true)) { return SetError(start, GetPosition(), "unterminated quoted identifier"); } return MakeToken(TokenType::kIdent, start, GetPosition()); @@ -582,7 +571,7 @@ Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes, return MakeToken(is_bytes ? TokenType::kBytes : TokenType::kString, start, GetPosition()); } - if (is_raw ? !ConsumeUntilAfter(quote) : !ConsumeUntilAfterUnescaped(quote)) { + if (!ConsumeUntilAfter(quote, is_raw)) { return SetError(start, GetPosition(), is_bytes ? "unterminated bytes literal" : "unterminated string literal"); diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h index 502be627a..70ac703cb 100644 --- a/parser/internal/lexer.h +++ b/parser/internal/lexer.h @@ -203,9 +203,10 @@ class Lexer final { } // Consumes characters up to and including the first occurrence of character - // `c` without interpreting backslashes as escapes. Returns true if `c` was - // found and consumed; false if end of input was reached. - [[nodiscard]] bool ConsumeUntilAfter(char32_t c); + // `c`. If `is_raw` is false, backslashes are interpreted as escapes. Returns + // true if `c` was found and consumed; false if end of input or an unescaped + // newline was reached. + [[nodiscard]] bool ConsumeUntilAfter(char32_t c, bool is_raw); // Consumes characters up to and including the first occurrence of substring // `s` without interpreting backslashes as escapes (`s` must not contain @@ -213,11 +214,6 @@ class Lexer final { // input was reached. [[nodiscard]] bool ConsumeUntilAfterString(std::u32string_view s); - // Consumes characters up to and including the first occurrence of `c` that is - // not preceded by an odd number of backslash ('\') escape characters. Returns - // true if an unescaped `c` was found and consumed; false if reached EOF. - [[nodiscard]] bool ConsumeUntilAfterUnescaped(char32_t c); - // Consumes characters up to and including the first occurrence of substring // `s` where the first character of `s` is not preceded by an odd number of // backslashes. Returns true if an unescaped `s` was found and consumed; false diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc index 9cdfbd45d..220c2df48 100644 --- a/parser/internal/lexer_test.cc +++ b/parser/internal/lexer_test.cc @@ -335,29 +335,23 @@ TEST(LexerTest, LineOffsets) { EXPECT_EQ(line_offsets[1], 13); } -TEST(LexerTest, LineOffsetsInStringsAndIdentifiers) { - std::string_view source_text = - "'''multi\nline'''\n\"another\nline\"\n`ident\nhere`"; +TEST(LexerTest, LineOffsetsInMultilineStrings) { + std::string_view source_text = "'''multi\nline'''\n\"\"\"another\nline\"\"\""; ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource(source_text)); Lexer lexer(*source); EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kString, "'''multi\nline'''")); EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); - EXPECT_THAT(lexer.Lex(), - IsToken(source.get(), TokenType::kString, "\"another\nline\"")); - EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kWhitespace, "\n")); - EXPECT_THAT(lexer.Lex(), - IsToken(source.get(), TokenType::kIdent, "`ident\nhere`")); + EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kString, + "\"\"\"another\nline\"\"\"")); EXPECT_THAT(lexer.Lex(), IsToken(source.get(), TokenType::kEnd, "")); auto line_offsets = source->line_offsets(); - ASSERT_GE(line_offsets.size(), 5); + ASSERT_GE(line_offsets.size(), 3); EXPECT_EQ(line_offsets[0], 9); EXPECT_EQ(line_offsets[1], 17); - EXPECT_EQ(line_offsets[2], 26); - EXPECT_EQ(line_offsets[3], 32); - EXPECT_EQ(line_offsets[4], 39); + EXPECT_EQ(line_offsets[2], 28); } struct LexerErrorTestCase { @@ -393,6 +387,12 @@ INSTANTIATE_TEST_SUITE_P( .expected_error_location = "\n | \"unterminated" "\n | .............^", }, + LexerErrorTestCase{ + .source = "\"another\nline\"", + .expected_error_message = "unterminated string literal", + .expected_error_location = "\n | \"another" + "\n | ........^", + }, LexerErrorTestCase{ .source = "0x", .expected_error_message = @@ -440,6 +440,12 @@ INSTANTIATE_TEST_SUITE_P( .expected_error_location = "\n | `unterminated quoted" "\n | ....................^", }, + LexerErrorTestCase{ + .source = "`ident\nhere`", + .expected_error_message = "unterminated quoted identifier", + .expected_error_location = "\n | `ident" + "\n | ......^", + }, LexerErrorTestCase{ .source = "'''unterminated multi", .expected_error_message = "unterminated string literal", diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 4fd2b02ec..fc6477745 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -361,6 +361,36 @@ std::vector GetParserTestCases() { "\x07\x08\x0c\n\r\t\x0b'\"\\? Legal escapes"^#1:string# )", }, + TestCase{ + .source = "'''hello\nworld'''", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "\"\"\"hello\nworld\"\"\"", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "r\"\"\"hello\nworld\"\"\"", + .expected_ast = R"( + "hello\nworld"^#1:string# + )", + }, + TestCase{ + .source = "\"\"\"hello\\\"\"\"world\"\"\"", + .expected_ast = R"( + "hello\"\"\"world"^#1:string# + )", + }, + TestCase{ + .source = "'''hello\\'''world'''", + .expected_ast = R"( + "hello'''world"^#1:string# + )", + }, TestCase{ .source = "a", .expected_ast = R"( @@ -1394,6 +1424,84 @@ std::vector GetErrorTestCases() { " | \"unterminated\n" " | ^", }, + ErrorTestCase{ + .source = "\"\"\"hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"\"\"hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "'''hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | '''hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "r\"\"\"hello\nworld", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"\"\"hello\n" + " | ^", + }, + ErrorTestCase{ + .source = "\"hello\nworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world\"\n" + " | ^", + }, + ErrorTestCase{ + .source = "'hello\nworld'", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world'\n" + " | ^", + }, + ErrorTestCase{ + .source = "r\"hello\nworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world\"\n" + " | ^", + }, + ErrorTestCase{ + .source = "`hello\nworld`", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated quoted " + "identifier\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after " + "expression\n" + " | world`\n" + " | ^", + .enable_quoted_identifiers = true, + }, + ErrorTestCase{ + .source = "\"hello\rworld\"", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:8: Syntax error: unexpected token after " + "expression\n" + " | \"hello\rworld\"\n" + " | .......^", + }, ErrorTestCase{ .source = "b\"unterminated", .expected_error = diff --git a/parser/parser_test.cc b/parser/parser_test.cc index d51339796..922968d3e 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1517,7 +1517,132 @@ std::vector test_cases = { "m^#5:Expr.Ident#.value()^#8:Expr.Call#,\n // LoopCondition\n " "false^#9:bool#,\n // LoopStep\n v^#3:Expr.Ident#,\n // Result\n " " f^#4:Expr.Ident#)^#10:Expr.Comprehension#,\n " - "optional.none()^#11:Expr.Call#\n)^#12:Expr.Call#"}}; + "optional.none()^#11:Expr.Call#\n)^#12:Expr.Call#"}, + + // Multiline and string literal tests + {"'''hello\nworld'''", "\"hello\\nworld\"^#1:string#"}, + {"\"\"\"hello\nworld\"\"\"", "\"hello\\nworld\"^#1:string#"}, + {"r\"\"\"hello\nworld\"\"\"", "\"hello\\nworld\"^#1:string#"}, + {"\"\"\"hello\\\"\"\"world\"\"\"", "\"hello\\\"\\\"\\\"world\"^#1:string#"}, + {"'''hello\\'''world'''", "\"hello'''world\"^#1:string#"}, + {"\"\"\"hello\nworld", "", + "ERROR: :1:3: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | \"\"\"hello\n" + " | ..^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"\"\"hello\n" + " | ^"}, + {"'''hello\nworld", "", + "ERROR: :1:3: Syntax error: token recognition error at: " + "''hello\\n'\n" + " | '''hello\n" + " | ..^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | '''hello\n" + " | ^"}, + {"r\"\"\"hello\nworld", "", + "ERROR: :1:4: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | r\"\"\"hello\n" + " | ...^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\n" + " | ^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"\"\"hello\n" + " | ^"}, + {"\"hello\nworld\"", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '\"'\n" + " | world\"\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world\"\n" + " | ^"}, + {"'hello\nworld'", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "''hello\\n'\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '''\n" + " | world'\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | 'hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world'\n" + " | ^"}, + {"r\"hello\nworld\"", "", + "ERROR: :1:2: Syntax error: token recognition error at: " + "'\"hello\\n'\n" + " | r\"hello\n" + " | .^\n" + "ERROR: :2:1: Syntax error: extraneous input 'world' expecting " + "\n" + " | world\"\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '\"'\n" + " | world\"\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | r\"hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world\"\n" + " | ^"}, + {"`hello\nworld`", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'`hello\\n'\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:6: Syntax error: token recognition error at: '`'\n" + " | world`\n" + " | .....^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated quoted identifier\n" + " | `hello\n" + " | ^\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" + " | world`\n" + " | ^"}, + {"\"hello\rworld\"", "", + "ERROR: :1:1: Syntax error: token recognition error at: " + "'\"hello\\r'\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:13: Syntax error: token recognition error at: '\"'\n" + " | \"hello\rworld\"\n" + " | ............^", + "", "", "", + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"hello\rworld\"\n" + " | ^\n" + "ERROR: :1:8: Syntax error: unexpected token after expression\n" + " | \"hello\rworld\"\n" + " | .......^"}}; absl::string_view ConstantKind(const cel::Constant& c) { switch (c.kind_case()) { From 8d22c5b66f28dc5f72b99b5c70fabd7587a592f2 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 15 Sep 2026 20:49:21 -0700 Subject: [PATCH 221/350] No public description PiperOrigin-RevId: 982227083 --- common/values/parsed_map_field_value_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/values/parsed_map_field_value_test.cc b/common/values/parsed_map_field_value_test.cc index c1a968646..0fdea333b 100644 --- a/common/values/parsed_map_field_value_test.cc +++ b/common/values/parsed_map_field_value_test.cc @@ -121,7 +121,7 @@ TEST_F(ParsedMapFieldValueTest, SerializeTo) { EXPECT_THAT(value.SerializeTo(descriptor_pool(), message_factory(), &output), IsOk()); google::protobuf::Struct actual; - EXPECT_TRUE(actual.ParsePartialFromCord(std::move(output).Consume())); + EXPECT_TRUE(actual.ParsePartialFromString(std::move(output).Consume())); EXPECT_THAT(actual, EqualsTextProto( R"pb(fields { key: "foo" From 2f44d1b8fd90c0ce806bf186bc8ce514159296dc Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 16 Sep 2026 14:43:11 -0700 Subject: [PATCH 222/350] Re-enable typed field accesses by default. PiperOrigin-RevId: 982734293 --- eval/public/cel_options.h | 2 +- runtime/runtime_options.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index a6e2692d4..32474e761 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -222,7 +222,7 @@ struct InterpreterOptions { // When enabled, the planner will attempt to use a more performant execution // path for field access when the type is known at plan time, instead of using // the generic field access implementation. - bool enable_typed_field_access = false; + bool enable_typed_field_access = true; // Temporary flag to gate using a new field selection implementation for // protos. diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 831183dc8..bc6275afb 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -200,7 +200,7 @@ struct RuntimeOptions { // This is not recommended if the values bound to the activation are typically // not what the planner expected (e.g. a map that was declared as a proto or // a different message with matching field names). - bool enable_typed_field_access = false; + bool enable_typed_field_access = true; // Temporary flag to gate using a new field selection implementation for // protos. From 789db8567b7a879106cc2c48d77746634b9576ef Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 17 Sep 2026 15:28:51 -0700 Subject: [PATCH 223/350] Temporarily revert same message factory checks. PiperOrigin-RevId: 983456583 --- common/values/struct_value_builder.cc | 13 +-- extensions/protobuf/value.h | 111 ++++++++------------------ extensions/protobuf/value_test.cc | 2 + 3 files changed, 37 insertions(+), 89 deletions(-) diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index cafdbeb54..9cbdf83ac 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -85,17 +85,8 @@ absl::StatusOr> ProtoMessageCopy( const google::protobuf::Message* absl_nonnull from_message) { CEL_ASSIGN_OR_RETURN(const auto* from_descriptor, GetDescriptor(*from_message)); - if (to_descriptor == from_descriptor && - to_message->GetReflection()->GetMessageFactory() == - from_message->GetReflection()->GetMessageFactory()) { - // Same type, use proto reflection copy. - // - // We use the slower serialization copy if the factory is different to avoid - // adding an implicit lifetime dependency on the other factory. - // - // This should only happen if the embedding application is calling the - // builder directly or attempting to set the field from an unsafe wrapped - // message. + if (to_descriptor == from_descriptor) { + // Same. to_message->CopyFrom(*from_message); return std::nullopt; } diff --git a/extensions/protobuf/value.h b/extensions/protobuf/value.h index 68049787f..4336b3d68 100644 --- a/extensions/protobuf/value.h +++ b/extensions/protobuf/value.h @@ -40,70 +40,6 @@ namespace cel::extensions { -namespace extensions_internal { -template -absl::Status ProtoMessageFromValue(const cel::Value& value, - google::protobuf::Message& dest_message) { - const auto* dest_descriptor = dest_message.GetDescriptor(); - const google::protobuf::Message* src_message = nullptr; - if (auto legacy_struct_value = - cel::common_internal::AsLegacyStructValue(value); - legacy_struct_value) { - src_message = legacy_struct_value->message_ptr(); - } - if (auto parsed_message_value = value.AsParsedMessage(); - parsed_message_value) { - src_message = cel::to_address(*parsed_message_value); - } - - if (src_message == nullptr) { - return TypeConversionError(value.GetRuntimeType(), - MessageType(dest_descriptor)) - .NativeValue(); - } - - const auto* src_descriptor = src_message->GetDescriptor(); - if (dest_descriptor != src_descriptor) { - goto slow_path; - } - - if constexpr (!SkipCopyCheck::value) { - // Try to catch cases where we'll take an implicit dependency on a - // dynamic message factory. - // - // This isn't exhaustive, but correctly checking requires fully - // traversing the source message which will approach the cost of the - // serialization round trip. - if (dest_message.GetReflection()->GetMessageFactory() != - src_message->GetReflection()->GetMessageFactory()) { - goto slow_path; - } - } - - dest_message.CopyFrom(*src_message); - return absl::OkStatus(); - -slow_path: - if (dest_descriptor->full_name() != src_descriptor->full_name()) { - return TypeConversionError(value.GetRuntimeType(), - MessageType(dest_descriptor)) - .NativeValue(); - } - - absl::Cord serialized; - if (!src_message->SerializePartialToCord(&serialized)) { - return absl::UnknownError(absl::StrCat("failed to serialize message: ", - src_descriptor->full_name())); - } - if (!dest_message.ParsePartialFromCord(serialized)) { - return absl::UnknownError(absl::StrCat("failed to parse message: ", - dest_descriptor->full_name())); - } - return absl::OkStatus(); -} - -} // namespace extensions_internal - // Adapt a protobuf message to a cel::Value. // // Handles unwrapping message types with special meanings in CEL (WKTs). @@ -123,20 +59,39 @@ ProtoMessageToValue(T&& value, // Unwraps a protobuf message from a cel::Value. inline absl::Status ProtoMessageFromValue(const Value& value, google::protobuf::Message& dest_message) { - return extensions_internal::ProtoMessageFromValue( - value, dest_message); -} - -// Unwraps a protobuf message from a cel::Value without checking for the -// presence of extensions. -// -// Warning: This function can lead to subtle use after free bugs if the caller -// is not careful to ensure that the source and destination message were created -// in a compatible way and do not outlive any implicit dependencies. -inline absl::Status ProtoMessageFromValueUnsafe(const Value& value, - google::protobuf::Message& dest_message) { - return extensions_internal::ProtoMessageFromValue( - value, dest_message); + const auto* dest_descriptor = dest_message.GetDescriptor(); + const google::protobuf::Message* src_message = nullptr; + if (auto legacy_struct_value = + cel::common_internal::AsLegacyStructValue(value); + legacy_struct_value) { + src_message = legacy_struct_value->message_ptr(); + } + if (auto parsed_message_value = value.AsParsedMessage(); + parsed_message_value) { + src_message = cel::to_address(*parsed_message_value); + } + if (src_message != nullptr) { + const auto* src_descriptor = src_message->GetDescriptor(); + if (dest_descriptor == src_descriptor) { + dest_message.CopyFrom(*src_message); + return absl::OkStatus(); + } + if (dest_descriptor->full_name() == src_descriptor->full_name()) { + absl::Cord serialized; + if (!src_message->SerializePartialToCord(&serialized)) { + return absl::UnknownError(absl::StrCat("failed to serialize message: ", + src_descriptor->full_name())); + } + if (!dest_message.ParsePartialFromCord(serialized)) { + return absl::UnknownError(absl::StrCat("failed to parse message: ", + dest_descriptor->full_name())); + } + return absl::OkStatus(); + } + } + return TypeConversionError(value.GetRuntimeType(), + MessageType(dest_descriptor)) + .NativeValue(); } } // namespace cel::extensions diff --git a/extensions/protobuf/value_test.cc b/extensions/protobuf/value_test.cc index cc89588f7..79c15690e 100644 --- a/extensions/protobuf/value_test.cc +++ b/extensions/protobuf/value_test.cc @@ -887,6 +887,8 @@ std::unique_ptr MakeTestExtendedMessage( } TEST_F(ProtoValueUnwrapTest, DynamicMessageFromUnderlayDescriptorPool) { + GTEST_SKIP() << "TODO(b/562935074): Avoid use-after-free when unwrapping " + "messages with extensions from a different factory."; const auto& pool = GetTestExternalExtensionsDescriptorPoolUnderlay(); TestAllTypes dest; { From 31221a72fb3ef2b6380a508882d8b32a1f0a349f Mon Sep 17 00:00:00 2001 From: Justin King Date: Thu, 17 Sep 2026 16:14:25 -0700 Subject: [PATCH 224/350] Prepare `cel::ErrorValue` to be trivial PiperOrigin-RevId: 983482153 --- common/value.h | 3 +- common/values/error_value.cc | 108 +++++++++++++++++++++++------- common/values/error_value.h | 125 +++++++++++++++++++++++++++-------- 3 files changed, 183 insertions(+), 53 deletions(-) diff --git a/common/value.h b/common/value.h index c2ca1c608..f6ce5de1a 100644 --- a/common/value.h +++ b/common/value.h @@ -2609,7 +2609,8 @@ static_assert(std::is_nothrow_swappable_v); inline common_internal::ImplicitlyConvertibleStatus ErrorValueAssign::operator()(absl::Status status) const { - *value_ = ErrorValue(std::move(status)); + *value_ = arena_ != nullptr ? ErrorValue::From(std::move(status), arena_) + : ErrorValue(std::move(status)); return common_internal::ImplicitlyConvertibleStatus(); } diff --git a/common/values/error_value.cc b/common/values/error_value.cc index 8ea6554ec..538ac4234 100644 --- a/common/values/error_value.cc +++ b/common/values/error_value.cc @@ -46,46 +46,106 @@ const absl::Status& DefaultErrorValue() { return *value; } +absl::Status MakeNoSuchFieldError(absl::string_view field) { + return absl::NotFoundError( + absl::StrCat("no_such_field", field.empty() ? "" : " : ", field)); +} + +absl::Status MakeNoSuchKeyError(absl::string_view key) { + return absl::NotFoundError(absl::StrCat("Key not found in map : ", key)); +} + +absl::Status MakeNoSuchTypeError(absl::string_view type) { + return absl::NotFoundError(absl::StrCat("type not found: ", type)); +} + +absl::Status MakeTypeConversionError(absl::string_view from, + absl::string_view to) { + return absl::InvalidArgumentError( + absl::StrCat("type conversion error from '", from, "' to '", to, "'")); +} + +absl::Status MakeIndexOutOfBoundsError(size_t index) { + return absl::InvalidArgumentError( + absl::StrCat("index out of bounds: ", index)); +} + +absl::Status MakeIndexOutOfBoundsError(ptrdiff_t index) { + return absl::InvalidArgumentError( + absl::StrCat("index out of bounds: ", index)); +} + } // namespace -ErrorValue::ErrorValue() : ErrorValue(DefaultErrorValue()) {} +ErrorValue::ErrorValue() : ErrorValue(nullptr, &DefaultErrorValue()) {} ErrorValue NoSuchFieldError(absl::string_view field) { - return ErrorValue(absl::NotFoundError( - absl::StrCat("no_such_field", field.empty() ? "" : " : ", field))); + return ErrorValue(MakeNoSuchFieldError(field)); +} + +ErrorValue NoSuchFieldError(absl::string_view field, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeNoSuchFieldError(field), arena); } ErrorValue NoSuchKeyError(absl::string_view key) { - return ErrorValue( - absl::NotFoundError(absl::StrCat("Key not found in map : ", key))); + return ErrorValue(MakeNoSuchKeyError(key)); +} + +ErrorValue NoSuchKeyError(absl::string_view key, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeNoSuchKeyError(key), arena); } ErrorValue NoSuchTypeError(absl::string_view type) { - return ErrorValue( - absl::NotFoundError(absl::StrCat("type not found: ", type))); + return ErrorValue(MakeNoSuchTypeError(type)); +} + +ErrorValue NoSuchTypeError(absl::string_view type, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeNoSuchTypeError(type), arena); } ErrorValue DuplicateKeyError() { - return ErrorValue(absl::AlreadyExistsError("duplicate key in map")); + static const absl::NoDestructor error( + absl::AlreadyExistsError("duplicate key in map")); + return ErrorValue(nullptr, &*error); } ErrorValue TypeConversionError(absl::string_view from, absl::string_view to) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("type conversion error from '", from, "' to '", to, "'"))); + return ErrorValue(MakeTypeConversionError(from, to)); +} + +ErrorValue TypeConversionError(absl::string_view from, absl::string_view to, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeTypeConversionError(from, to), arena); } ErrorValue TypeConversionError(const Type& from, const Type& to) { return TypeConversionError(from.DebugString(), to.DebugString()); } +ErrorValue TypeConversionError(const Type& from, const Type& to, + google::protobuf::Arena* absl_nonnull arena) { + return TypeConversionError(from.DebugString(), to.DebugString(), arena); +} + ErrorValue IndexOutOfBoundsError(size_t index) { - return ErrorValue( - absl::InvalidArgumentError(absl::StrCat("index out of bounds: ", index))); + return ErrorValue(MakeIndexOutOfBoundsError(index)); +} + +ErrorValue IndexOutOfBoundsError(size_t index, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeIndexOutOfBoundsError(index), arena); } ErrorValue IndexOutOfBoundsError(ptrdiff_t index) { - return ErrorValue( - absl::InvalidArgumentError(absl::StrCat("index out of bounds: ", index))); + return ErrorValue(MakeIndexOutOfBoundsError(index)); +} + +ErrorValue IndexOutOfBoundsError(ptrdiff_t index, + google::protobuf::Arena* absl_nonnull arena) { + return ErrorValue::From(MakeIndexOutOfBoundsError(index), arena); } bool IsNoSuchField(const ErrorValue& value) { @@ -159,30 +219,28 @@ ErrorValue ErrorValue::Clone(google::protobuf::Arena* absl_nonnull arena) const absl::Status ErrorValue::ToStatus() const& { ABSL_DCHECK(*this); - - if (arena_ == nullptr) { + if (status_ptr_ == nullptr) { return *std::launder( - reinterpret_cast(&status_.val[0])); + reinterpret_cast(&status_val_[0])); } - return *status_.ptr; + return *status_ptr_; } absl::Status ErrorValue::ToStatus() && { ABSL_DCHECK(*this); - - if (arena_ == nullptr) { + if (status_ptr_ == nullptr) { return std::move( - *std::launder(reinterpret_cast(&status_.val[0]))); + *std::launder(reinterpret_cast(&status_val_[0]))); } - return *status_.ptr; + return *status_ptr_; } ErrorValue::operator bool() const { - if (arena_ == nullptr) { - return !std::launder(reinterpret_cast(&status_.val[0])) + if (status_ptr_ == nullptr) { + return !std::launder(reinterpret_cast(&status_val_[0])) ->ok(); } - return status_.ptr != nullptr && !status_.ptr->ok(); + return !status_ptr_->ok(); } void swap(ErrorValue& lhs, ErrorValue& rhs) noexcept { diff --git a/common/values/error_value.h b/common/values/error_value.h index 4e24c866b..7008df7fa 100644 --- a/common/values/error_value.h +++ b/common/values/error_value.h @@ -31,7 +31,6 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "common/arena.h" #include "common/type.h" @@ -45,6 +44,9 @@ namespace cel { class Value; +class ErrorValue; + +ErrorValue DuplicateKeyError(); // `ErrorValue` represents values of the `ErrorType`. class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final @@ -52,8 +54,22 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final public: static constexpr ValueKind kKind = ValueKind::kError; - explicit ErrorValue(absl::Status value) : arena_(nullptr) { - ::new (static_cast(&status_.val[0])) absl::Status(std::move(value)); + // Returns a new ErrorValue created from absl::Status. The resulting storage + // for the underlying representation of ErrorValue is stored on the arena. + [[nodiscard]] + static ErrorValue From(absl::Status value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + ABSL_DCHECK(!value.ok()) << "ErrorValue requires a non-OK absl::Status"; + ABSL_DCHECK(arena != nullptr); + return ErrorValue( + arena, google::protobuf::Arena::Create(arena, std::move(value))); + } + + ABSL_DEPRECATED("Use From") + explicit ErrorValue(absl::Status value) + : arena_(nullptr), status_ptr_(nullptr) { + ::new (static_cast(&status_val_[0])) absl::Status(std::move(value)); ABSL_DCHECK(*this) << "ErrorValue requires a non-OK absl::Status"; } @@ -127,80 +143,112 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final explicit operator bool() const; private: + friend ErrorValue DuplicateKeyError(); friend class common_internal::ValueMixin; friend struct ArenaTraits; - ErrorValue(google::protobuf::Arena* absl_nonnull arena, + ErrorValue(google::protobuf::Arena* absl_nullable arena, const absl::Status* absl_nonnull status) - : arena_(arena) { - status_.ptr = status; - } + : arena_(arena), status_ptr_(status) {} void CopyConstruct(const ErrorValue& other) { arena_ = other.arena_; - if (arena_ == nullptr) { - ::new (static_cast(&status_.val[0])) absl::Status(*std::launder( - reinterpret_cast(&other.status_.val[0]))); - } else { - status_.ptr = other.status_.ptr; + status_ptr_ = other.status_ptr_; + if (status_ptr_ == nullptr) { + ::new (static_cast(&status_val_[0])) absl::Status(*std::launder( + reinterpret_cast(&other.status_val_[0]))); } } void MoveConstruct(ErrorValue& other) { arena_ = other.arena_; - if (arena_ == nullptr) { - ::new (static_cast(&status_.val[0])) + status_ptr_ = other.status_ptr_; + if (status_ptr_ == nullptr) { + ::new (static_cast(&status_val_[0])) absl::Status(std::move(*std::launder( - reinterpret_cast(&other.status_.val[0])))); - } else { - status_.ptr = other.status_.ptr; + reinterpret_cast(&other.status_val_[0])))); } } void Destruct() { - if (arena_ == nullptr) { - std::launder(reinterpret_cast(&status_.val[0]))->~Status(); + if (status_ptr_ == nullptr) { + std::launder(reinterpret_cast(&status_val_[0]))->~Status(); } } google::protobuf::Arena* absl_nullable arena_; - union { - alignas(absl::Status) char val[sizeof(absl::Status)]; - const absl::Status* absl_nonnull ptr; - } status_; + const absl::Status* absl_nullable status_ptr_ = nullptr; + alignas(absl::Status) char status_val_[sizeof(absl::Status)]; }; +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") ErrorValue NoSuchFieldError(absl::string_view field); +ErrorValue NoSuchFieldError(absl::string_view field, + google::protobuf::Arena* absl_nonnull arena); +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") ErrorValue NoSuchKeyError(absl::string_view key); +ErrorValue NoSuchKeyError(absl::string_view key, + google::protobuf::Arena* absl_nonnull arena); +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") ErrorValue NoSuchTypeError(absl::string_view type); +ErrorValue NoSuchTypeError(absl::string_view type, + google::protobuf::Arena* absl_nonnull arena); ErrorValue DuplicateKeyError(); +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") ErrorValue TypeConversionError(absl::string_view from, absl::string_view to); +ErrorValue TypeConversionError(absl::string_view from, absl::string_view to, + google::protobuf::Arena* absl_nonnull arena); +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") ErrorValue TypeConversionError(const Type& from, const Type& to); +ErrorValue TypeConversionError(const Type& from, const Type& to, + google::protobuf::Arena* absl_nonnull arena); +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") ErrorValue IndexOutOfBoundsError(size_t index); +ErrorValue IndexOutOfBoundsError(size_t index, + google::protobuf::Arena* absl_nonnull arena); +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") ErrorValue IndexOutOfBoundsError(ptrdiff_t index); +ErrorValue IndexOutOfBoundsError(ptrdiff_t index, + google::protobuf::Arena* absl_nonnull arena); // Catch other integrals and forward them to the above ones. This is needed to // avoid ambiguous overload issues for smaller integral types like `int`. template +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") +std::enable_if_t, std::is_unsigned, + std::negation>>, + ErrorValue> IndexOutOfBoundsError(T index) { + static_assert(sizeof(T) <= sizeof(size_t)); + return IndexOutOfBoundsError(static_cast(index)); +} +template std::enable_if_t, std::is_unsigned, std::negation>>, ErrorValue> -IndexOutOfBoundsError(T index) { +IndexOutOfBoundsError(T index, google::protobuf::Arena* absl_nonnull arena) { static_assert(sizeof(T) <= sizeof(size_t)); return IndexOutOfBoundsError(static_cast(index)); } template +ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") +std::enable_if_t, std::is_signed, + std::negation>>, + ErrorValue> IndexOutOfBoundsError(T index) { + static_assert(sizeof(T) <= sizeof(ptrdiff_t)); + return IndexOutOfBoundsError(static_cast(index)); +} +template std::enable_if_t, std::is_signed, std::negation>>, ErrorValue> -IndexOutOfBoundsError(T index) { +IndexOutOfBoundsError(T index, google::protobuf::Arena* absl_nonnull arena) { static_assert(sizeof(T) <= sizeof(ptrdiff_t)); return IndexOutOfBoundsError(static_cast(index)); } @@ -215,11 +263,20 @@ bool IsNoSuchKey(const ErrorValue& value); class ErrorValueReturn final { public: + ABSL_DEPRECATED("Use constructor which takes google::protobuf::Arena*") ErrorValueReturn() = default; + explicit ErrorValueReturn(google::protobuf::Arena* absl_nonnull arena) : arena_(arena) { + ABSL_DCHECK(arena != nullptr); + } + ErrorValue operator()(absl::Status status) const { - return ErrorValue(std::move(status)); + return arena_ != nullptr ? ErrorValue::From(std::move(status), arena_) + : ErrorValue(std::move(status)); } + + private: + google::protobuf::Arena* arena_ = nullptr; }; namespace common_internal { @@ -248,26 +305,40 @@ class ErrorValueAssign final { public: ErrorValueAssign() = delete; + ABSL_DEPRECATED("Use constructor which takes google::protobuf::Arena*") explicit ErrorValueAssign(Value& value ABSL_ATTRIBUTE_LIFETIME_BOUND) : ErrorValueAssign(std::addressof(value)) {} + ABSL_DEPRECATED("Use constructor which takes google::protobuf::Arena*") explicit ErrorValueAssign( Value* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND) : value_(value) { ABSL_DCHECK(value != nullptr); } + ErrorValueAssign(Value& value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nonnull arena) + : ErrorValueAssign(std::addressof(value), arena) {} + + ErrorValueAssign(Value* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nonnull arena) + : value_(value), arena_(arena) { + ABSL_DCHECK(value != nullptr); + ABSL_DCHECK(arena != nullptr); + } + common_internal::ImplicitlyConvertibleStatus operator()( absl::Status status) const; private: Value* absl_nonnull value_; + google::protobuf::Arena* arena_ = nullptr; }; template <> struct ArenaTraits { static bool trivially_destructible(const ErrorValue& value) { - return value.arena_ != nullptr; + return value.status_ptr_ != nullptr; } }; From dd26ee7cd3a91f585ed1877e45dfcdf01196b7c6 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 17 Sep 2026 16:26:28 -0700 Subject: [PATCH 225/350] internal PiperOrigin-RevId: 983488246 --- bazel/BUILD | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bazel/BUILD b/bazel/BUILD index 5b3cb2d2c..5edddadda 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -19,7 +19,9 @@ exports_files( cc_binary( name = "cel_cc_embed", srcs = ["cel_cc_embed.cc"], - visibility = ["//:__subpackages__"], + visibility = [ + "//:__subpackages__", + ], deps = [ "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", @@ -33,7 +35,9 @@ cc_binary( cc_binary( name = "cat_param_file", srcs = ["cat_param_file.cc"], - visibility = ["//:__subpackages__"], + visibility = [ + "//:__subpackages__", + ], deps = [ "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", From 680d2829116d27c1fe55dafb68e15137bb291a26 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 18 Sep 2026 14:08:16 -0700 Subject: [PATCH 226/350] [Pratt Parser] Cache source content size in Lexer to avoid repeated variant dispatch in character loops. `SourceContentView::size()` is defined out-of-line and dispatches via `absl::visit` over a 4-element `absl::variant`. Because the character scanning loops in `Lexer` also invoke out-of-line methods on `content_`, the compiler cannot hoist `content_.size()` out of loop conditions, causing it to be re-evaluated on every character scanned. Caching `content_size_` in `Lexer` improves Pratt parser CPU time across `pratt_parser_benchmark` by ~5.2% geomean (~6.7% geomean on valid expressions). Measured with `pratt_parser_benchmark` (built `-c opt --dynamic_mode=off`). "Pratt before" is this CL's parent (`p4head`), so the last column is this CL's own contribution. The ANTLR column is the series baseline and is unaffected by this change. CPU time, median across 36 interleaved repetitions: | Case | ANTLR | Pratt before | Pratt after | Pratt vs ANTLR | Delta this CL | | :--- | ---: | ---: | ---: | ---: | ---: | | ParseCommon | 210,982 ns | 11,606 ns | 11,003 ns | 19.2x faster | -5.2% | | ParseArithmeticChain/10 | 55,289 ns | 2,781 ns | 2,608 ns | 21.2x faster | -6.2% | | ParseArithmeticChain/50 | 291,952 ns | 13,794 ns | 13,265 ns | 22.0x faster | -3.8% | | ParseArithmeticChain/100 | 559,317 ns | 27,506 ns | 25,724 ns | 21.7x faster | -6.5% | | ParseLogicalChain/10 | 43,318 ns | 3,306 ns | 2,986 ns | 14.5x faster | -9.7% | | ParseLogicalChain/50 | 207,612 ns | 15,118 ns | 13,940 ns | 14.9x faster | -7.8% | | ParseLogicalChain/100 | 442,381 ns | 30,089 ns | 28,675 ns | 15.4x faster | -4.7% | | ParseMemberChain/10 | 34,693 ns | 1,819 ns | 1,705 ns | 20.4x faster | -6.3% | | ParseMemberChain/50 | 161,104 ns | 8,633 ns | 7,889 ns | 20.4x faster | -8.6% | | ParseMemberChain/100 | 335,750 ns | 16,917 ns | 15,724 ns | 21.4x faster | -7.1% | | ParseNestedParentheses/10 | 28,084 ns | 803 ns | 752 ns | 37.3x faster | -6.3% | | ParseNestedParentheses/50 | 123,579 ns | 2,841 ns | 2,620 ns | 47.2x faster | -7.8% | | ParseCommonSyntaxErrors | 429,515 ns | 18,317 ns | 17,561 ns | 24.5x faster | -4.1% | | ParseArithmeticChainSyntaxError/10 | 72,212 ns | 3,256 ns | 3,078 ns | 23.5x faster | -5.5% | | ParseArithmeticChainSyntaxError/50 | 295,287 ns | 14,321 ns | 13,419 ns | 22.0x faster | -6.3% | | ParseArithmeticChainSyntaxError/100 | 628,385 ns | 28,006 ns | 26,414 ns | 23.8x faster | -5.7% | | ParseLogicalChainSyntaxError/10 | 58,855 ns | 3,776 ns | 3,553 ns | 16.6x faster | -5.9% | | ParseLogicalChainSyntaxError/50 | 235,134 ns | 16,104 ns | 15,147 ns | 15.5x faster | -5.9% | | ParseLogicalChainSyntaxError/100 | 458,219 ns | 31,111 ns | 29,472 ns | 15.5x faster | -5.3% | | ParseMemberChainSyntaxError/10 | 50,335 ns | 2,144 ns | 2,051 ns | 24.5x faster | -4.3% | | ParseMemberChainSyntaxError/50 | 182,649 ns | 8,962 ns | 8,516 ns | 21.4x faster | -5.0% | | ParseMemberChainSyntaxError/100 | 346,654 ns | 17,161 ns | 16,230 ns | 21.4x faster | -5.4% | | ParseNestedParenthesesSyntaxError/10 | 241,350 ns | 6,098 ns | 5,982 ns | 40.3x faster | -1.9% | | ParseNestedParenthesesSyntaxError/50 | 1,561,548 ns | 28,725 ns | 27,390 ns | 57.0x faster | -4.6% | | ParseRepeatedSyntaxErrors/10 | 170,609 ns | 6,218 ns | 6,044 ns | 28.2x faster | -2.8% | | ParseRepeatedSyntaxErrors/50 | 172,554 ns | 8,602 ns | 8,620 ns | 20.0x faster | +0.2% | | ParseRepeatedSyntaxErrors/100 | 180,960 ns | 9,324 ns | 9,532 ns | 19.0x faster | +2.2% | | **Geomean** | | | | **22.5x faster** | **-5.2%** | PiperOrigin-RevId: 984058754 --- parser/internal/lexer.cc | 43 ++++++++++++++++++++-------------------- parser/internal/lexer.h | 15 ++++++++------ 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc index 4163833be..dfc78e922 100644 --- a/parser/internal/lexer.cc +++ b/parser/internal/lexer.cc @@ -162,7 +162,7 @@ std::string_view TokenTypeToString(TokenType type) { Token Lexer::Lex() { int32_t start = GetPosition(); - if (ABSL_PREDICT_FALSE(position_ >= content_.size())) { + if (ABSL_PREDICT_FALSE(position_ >= content_size_)) { return MakeToken(TokenType::kEnd, start, start); } char32_t c = content_.at(position_); @@ -182,8 +182,7 @@ Token Lexer::Lex() { return MakeToken(TokenType::kWhitespace, start, GetPosition()); } case '.': { - if (position_ + 1 < content_.size() && - content_.at(position_ + 1) <= 0x7f && + if (position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f && absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { return ConsumeNumericLiteral(); } @@ -342,7 +341,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { ABSL_DCHECK_NE(c, '\r'); int32_t pos = position_; bool escaped = false; - while (pos < content_.size()) { + while (pos < content_size_) { char32_t cc = content_.at(pos); if (cc == '\n' || cc == '\r') { AdvanceProcessingNewLines(pos); @@ -359,7 +358,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } @@ -370,7 +369,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); int32_t pos = position_; - while (pos + static_cast(s.size()) <= content_.size()) { + while (pos + static_cast(s.size()) <= content_size_) { bool match = true; for (size_t i = 0; i < s.size(); ++i) { if (content_.at(pos + static_cast(i)) != s[i]) { @@ -384,7 +383,7 @@ bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } @@ -396,12 +395,12 @@ bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); int32_t pos = position_; bool escaped = false; - while (pos < content_.size()) { + while (pos < content_size_) { char32_t cc = content_.at(pos); if (cc == '\\') { escaped = !escaped; } else { - if (!escaped && pos + static_cast(s.size()) <= content_.size()) { + if (!escaped && pos + static_cast(s.size()) <= content_size_) { bool match = true; for (size_t j = 0; j < s.size(); ++j) { if (content_.at(pos + static_cast(j)) != s[j]) { @@ -418,12 +417,12 @@ bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } bool Lexer::MatchString(std::u32string_view s) const { - if (position_ + static_cast(s.size()) > content_.size()) { + if (position_ + static_cast(s.size()) > content_size_) { return false; } for (size_t i = 0; i < s.size(); ++i) { @@ -436,7 +435,7 @@ bool Lexer::MatchString(std::u32string_view s) const { std::optional Lexer::MatchIf( absl::FunctionRef predicate) const { - if (position_ < content_.size()) { + if (position_ < content_size_) { char32_t cp = content_.at(position_); if (predicate(cp)) { return cp; @@ -446,7 +445,7 @@ std::optional Lexer::MatchIf( } void Lexer::ConsumeLine() { - while (position_ < content_.size()) { + while (position_ < content_size_) { if (content_.at(position_) == '\n') { Advance(1); return; @@ -456,7 +455,7 @@ void Lexer::ConsumeLine() { } void Lexer::ConsumeWhitespace() { - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); switch (c) { case '\f': @@ -517,7 +516,7 @@ std::optional Lexer::ConsumeIf( bool Lexer::ConsumeDigits() { bool advanced = false; - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (c > 0x7f || !absl::ascii_isdigit(static_cast(c))) { break; @@ -530,7 +529,7 @@ bool Lexer::ConsumeDigits() { bool Lexer::ConsumeHexDigits() { bool advanced = false; - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (c > 0x7f || !absl::ascii_isxdigit(static_cast(c))) { break; @@ -588,12 +587,12 @@ Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes, // rb"""...""", rb'''...''' std::optional Lexer::ConsumePrefixedStringLiteral() { int32_t start = GetPosition(); - if (position_ >= content_.size()) return std::nullopt; + if (position_ >= content_size_) return std::nullopt; char32_t c = content_.at(position_); bool is_bytes = (c == 'b' || c == 'B'); bool is_raw = (c == 'r' || c == 'R'); size_t lookahead = 1; - if (position_ + 1 < content_.size()) { + if (position_ + 1 < content_size_) { char32_t c2 = content_.at(position_ + 1); if ((is_bytes && (c2 == 'r' || c2 == 'R')) || (!is_bytes && (c2 == 'b' || c2 == 'B'))) { @@ -602,7 +601,7 @@ std::optional Lexer::ConsumePrefixedStringLiteral() { lookahead = 2; } } - if (position_ + static_cast(lookahead) < content_.size()) { + if (position_ + static_cast(lookahead) < content_size_) { char32_t quote = content_.at(position_ + static_cast(lookahead)); if (quote == '"' || quote == '\'') { Advance(lookahead); @@ -649,8 +648,8 @@ Token Lexer::ConsumeNumericLiteral() { } } static_cast(ConsumeDigits()); - if (position_ < content_.size() && content_.at(position_) == '.' && - position_ + 1 < content_.size() && content_.at(position_ + 1) <= 0x7f && + if (position_ < content_size_ && content_.at(position_) == '.' && + position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f && absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { floating_point = true; Advance(1); @@ -679,7 +678,7 @@ Token Lexer::ConsumeNumericLiteral() { Token Lexer::ConsumeIdent() { int32_t start = GetPosition(); - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (!IsIdentTrailing(c)) { break; diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h index 70ac703cb..217bc7c41 100644 --- a/parser/internal/lexer.h +++ b/parser/internal/lexer.h @@ -135,7 +135,9 @@ struct LexerError final { class Lexer final { public: explicit Lexer(const cel::Source& source) - : content_(source.content()), position_(0) { + : content_(source.content()), + content_size_(static_cast(content_.size())), + position_(0) { ABSL_DCHECK_LE(content_.size(), static_cast( std::numeric_limits::max())); } @@ -162,18 +164,18 @@ class Lexer final { void RestorePosition(int32_t position) { ABSL_DCHECK_GE(position, 0); - ABSL_DCHECK_LE(position, static_cast(content_.size())); + ABSL_DCHECK_LE(position, content_size_); position_ = position; error_ = LexerError{}; } private: [[nodiscard]] bool Match(char32_t c) const { - return position_ < content_.size() && content_.at(position_) == c; + return position_ < content_size_ && content_.at(position_) == c; } [[nodiscard]] bool MatchIgnoreCase(char32_t c) const { - if (position_ >= content_.size()) return false; + if (position_ >= content_size_) return false; char32_t cp = content_.at(position_); return cp <= 0x7f && c <= 0x7f && absl::ascii_tolower(static_cast(cp)) == @@ -181,12 +183,12 @@ class Lexer final { } void Advance(size_t n) { - ABSL_DCHECK_LE(n, static_cast(content_.size() - position_)); + ABSL_DCHECK_LE(n, static_cast(content_size_ - position_)); position_ += static_cast(n); } void AdvanceProcessingNewLines(int32_t end_position) { - ABSL_DCHECK_LE(end_position, content_.size()); + ABSL_DCHECK_LE(end_position, content_size_); ABSL_DCHECK_GE(end_position, position_); Advance(static_cast(end_position - position_)); } @@ -275,6 +277,7 @@ class Lexer final { [[nodiscard]] Token ConsumeIdent(); cel::SourceContentView content_; + int32_t content_size_ = 0; int32_t position_ = 0; LexerError error_; }; From 354bca8f2625035066f151b381fc434392ff1b09 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Fri, 18 Sep 2026 16:50:22 -0700 Subject: [PATCH 227/350] Add helper for parsing a single test case from yaml. PiperOrigin-RevId: 984137239 --- policy/BUILD | 1 - policy/test_util.cc | 109 ++++++++++++++++++++++++++++++++------------ policy/test_util.h | 5 ++ 3 files changed, 86 insertions(+), 29 deletions(-) diff --git a/policy/BUILD b/policy/BUILD index 6a204d07c..9b7fe4202 100644 --- a/policy/BUILD +++ b/policy/BUILD @@ -225,7 +225,6 @@ cc_test( cc_library( name = "test_util", - testonly = True, srcs = ["test_util.cc"], hdrs = ["test_util.h"], copts = ["-fexceptions"], diff --git a/policy/test_util.cc b/policy/test_util.cc index 9fe1e43d1..a567e3862 100644 --- a/policy/test_util.cc +++ b/policy/test_util.cc @@ -10,9 +10,9 @@ #include "policy/test_util.h" #include +#include #include #include -#include #include "cel/expr/eval.pb.h" #include "cel/expr/value.pb.h" @@ -131,6 +131,73 @@ absl::Status ParseTestOutput(const YAML::Node& node, return YamlToExprValue(node, output->mutable_result_value()); } +absl::Status ParseTestCaseNode( + const YAML::Node& test_node, + cel::expr::conformance::test::TestCase* test_case) { + if (test_node.IsNull()) { + return absl::OkStatus(); + } + if (!test_node.IsMap()) { + return absl::InvalidArgumentError("Test case must be a YAML mapping"); + } + if (test_node["name"].IsDefined()) { + test_case->set_name(test_node["name"].as()); + } + if (test_node["description"].IsDefined()) { + test_case->set_description(test_node["description"].as()); + } + if (test_node["expr"].IsDefined()) { + test_case->set_expr(test_node["expr"].as()); + } + if (test_node["context_expr"].IsDefined()) { + test_case->mutable_input_context()->set_context_expr( + test_node["context_expr"].as()); + } else if (test_node["input_context"].IsDefined() && + test_node["input_context"].IsMap() && + test_node["input_context"]["context_expr"].IsDefined()) { + test_case->mutable_input_context()->set_context_expr( + test_node["input_context"]["context_expr"].as()); + } + if (test_node["disable_check"].IsDefined()) { + test_case->set_disable_check(test_node["disable_check"].as()); + } + + YAML::Node input_node = test_node["input"]; + if (input_node.IsDefined() && input_node.IsMap()) { + auto* input_map = test_case->mutable_input(); + for (auto it = input_node.begin(); it != input_node.end(); ++it) { + std::string var_name = it->first.as(); + cel::expr::conformance::test::InputValue input_val; + CEL_RETURN_IF_ERROR(ParseInputValue(it->second, &input_val)); + (*input_map)[var_name] = std::move(input_val); + } + } + + YAML::Node output_node = test_node["output"]; + if (output_node.IsDefined()) { + CEL_RETURN_IF_ERROR( + ParseTestOutput(output_node, test_case->mutable_output())); + } + return absl::OkStatus(); +} + +absl::StatusOr ParseTestCaseYamlImpl( + absl::string_view yaml_content) { + if (yaml_content.empty()) { + return cel::expr::conformance::test::TestCase(); + } + YAML::Node test_node; + try { + test_node = YAML::Load(std::string(yaml_content)); + } catch (const std::exception& e) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse YAML: ", e.what())); + } + cel::expr::conformance::test::TestCase test_case; + CEL_RETURN_IF_ERROR(ParseTestCaseNode(test_node, &test_case)); + return test_case; +} + absl::StatusOr ParsePolicyTestSuiteYamlImpl(absl::string_view yaml_content) { YAML::Node tests_node; @@ -174,33 +241,7 @@ ParsePolicyTestSuiteYamlImpl(absl::string_view yaml_content) { for (const auto& test_node : tests) { auto* test_case = section->add_tests(); - if (test_node["name"].IsDefined()) { - test_case->set_name(test_node["name"].as()); - } - if (test_node["description"].IsDefined()) { - test_case->set_description(test_node["description"].as()); - } - if (test_node["context_expr"].IsDefined()) { - test_case->mutable_input_context()->set_context_expr( - test_node["context_expr"].as()); - } - - YAML::Node input_node = test_node["input"]; - if (input_node.IsDefined() && input_node.IsMap()) { - auto* input_map = test_case->mutable_input(); - for (auto it = input_node.begin(); it != input_node.end(); ++it) { - std::string var_name = it->first.as(); - cel::expr::conformance::test::InputValue input_val; - CEL_RETURN_IF_ERROR(ParseInputValue(it->second, &input_val)); - (*input_map)[var_name] = std::move(input_val); - } - } - - YAML::Node output_node = test_node["output"]; - if (output_node.IsDefined()) { - CEL_RETURN_IF_ERROR( - ParseTestOutput(output_node, test_case->mutable_output())); - } + CEL_RETURN_IF_ERROR(ParseTestCaseNode(test_node, test_case)); } } @@ -218,4 +259,16 @@ ParsePolicyTestSuiteYaml(absl::string_view yaml_content) { } } +absl::StatusOr ParseTestCaseYaml( + absl::string_view yaml_content) { + try { + return ParseTestCaseYamlImpl(yaml_content); + } catch (const std::exception& e) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse YAML: ", e.what())); + } catch (...) { + return absl::InvalidArgumentError("Failed to parse YAML"); + } +} + } // namespace cel::test diff --git a/policy/test_util.h b/policy/test_util.h index 5fe306050..c917a2581 100644 --- a/policy/test_util.h +++ b/policy/test_util.h @@ -28,6 +28,11 @@ namespace cel::test { absl::StatusOr ParsePolicyTestSuiteYaml(absl::string_view yaml_content); +// Parses a YAML string representing an individual test case and adapts it to +// the cel.expr.conformance.test.TestCase protobuf message. +absl::StatusOr ParseTestCaseYaml( + absl::string_view yaml_content); + } // namespace cel::test #endif // THIRD_PARTY_CEL_CPP_POLICY_TEST_UTIL_H_ From 065815b3781920b899f8d1a6fa6decdbc30f9350 Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 21 Sep 2026 11:57:52 -0700 Subject: [PATCH 228/350] Make `cel::AttributeSet` copy-on-write PiperOrigin-RevId: 985421843 --- base/BUILD | 1 + base/attribute_set.h | 66 +++++++++++++++++++------- common/legacy_value.cc | 10 ++-- common/values/unknown_value.h | 60 +++++++++++++++-------- eval/eval/attribute_utility.cc | 42 +++++++++------- eval/eval/attribute_utility_test.cc | 14 +++--- eval/eval/create_list_step_test.cc | 16 ++++--- eval/eval/ident_step_test.cc | 2 +- eval/eval/logic_step_test.cc | 3 +- eval/eval/select_step_test.cc | 7 +-- eval/eval/ternary_step_test.cc | 6 +-- extensions/select_optimization_test.cc | 4 +- 12 files changed, 151 insertions(+), 80 deletions(-) diff --git a/base/BUILD b/base/BUILD index a239d4751..8c2a32671 100644 --- a/base/BUILD +++ b/base/BUILD @@ -35,6 +35,7 @@ cc_library( ":kind", "//internal:status_macros", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:btree", "@com_google_absl//absl/status", diff --git a/base/attribute_set.h b/base/attribute_set.h index 078f37881..625893fd9 100644 --- a/base/attribute_set.h +++ b/base/attribute_set.h @@ -15,6 +15,9 @@ #ifndef THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_SET_H_ #define THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_SET_H_ +#include + +#include "absl/base/no_destructor.h" #include "absl/container/btree_set.h" #include "absl/types/span.h" #include "base/attribute.h" @@ -54,27 +57,43 @@ class AttributeSet final { } } - AttributeSet(const AttributeSet& set1, const AttributeSet& set2) - : attributes_(set1.attributes_) { - for (const auto& attr : set2.attributes_) { - Add(attr); - } + AttributeSet(const AttributeSet& set1, const AttributeSet& set2) { + Add(set1); + Add(set2); } - iterator begin() const { return attributes_.begin(); } + iterator begin() const { + return attributes_ != nullptr ? attributes_->begin() + : EmptyContainer().begin(); + } - const_iterator cbegin() const { return attributes_.cbegin(); } + const_iterator cbegin() const { + return attributes_ != nullptr ? attributes_->cbegin() + : EmptyContainer().cbegin(); + } - iterator end() const { return attributes_.end(); } + iterator end() const { + return attributes_ != nullptr ? attributes_->end() : EmptyContainer().end(); + } - const_iterator cend() const { return attributes_.cend(); } + const_iterator cend() const { + return attributes_ != nullptr ? attributes_->cend() + : EmptyContainer().cend(); + } - size_type size() const { return attributes_.size(); } + size_type size() const { + return attributes_ != nullptr ? attributes_->size() : 0; + } - bool empty() const { return attributes_.empty(); } + bool empty() const { return attributes_ == nullptr || attributes_->empty(); } bool operator==(const AttributeSet& other) const { - return this == &other || attributes_ == other.attributes_; + if (attributes_ == other.attributes_) { + return true; + } + return (attributes_ != nullptr ? *attributes_ : EmptyContainer()) == + (other.attributes_ != nullptr ? *other.attributes_ + : EmptyContainer()); } bool operator!=(const AttributeSet& other) const { @@ -91,16 +110,31 @@ class AttributeSet final { friend class UnknownValue; friend class base_internal::UnknownSet; - void Add(const Attribute& attribute) { attributes_.insert(attribute); } + static const Container& EmptyContainer() { + static const absl::NoDestructor container; + return *container; + } + + void Add(const Attribute& attribute) { + if (attributes_ == nullptr) { + attributes_ = std::make_shared(); + } + attributes_->insert(attribute); + } void Add(const AttributeSet& other) { - for (const auto& attribute : other) { - Add(attribute); + if (!other.empty()) { + if (attributes_ == nullptr) { + attributes_ = std::make_shared(); + } + for (const auto& attribute : other) { + attributes_->insert(attribute); + } } } // Attribute container. - Container attributes_; + std::shared_ptr attributes_; }; } // namespace cel diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 3a2108c4d..c2dc1cc7f 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -1073,7 +1073,8 @@ absl::Status ModernValue(google::protobuf::Arena* arena, MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); return absl::OkStatus(); case CelValue::Type::kUnknownSet: - result = UnknownValue{*legacy_value.UnknownSetOrDie()}; + result = + common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie()); return absl::OkStatus(); case CelValue::Type::kCelType: { auto type_name = legacy_value.CelTypeOrDie().value(); @@ -1132,7 +1133,8 @@ absl::StatusOr LegacyValue( return common_internal::LegacyTrivialMapValue(arena, modern_value); case ValueKind::kUnknown: return CelValue::CreateUnknownSet(google::protobuf::Arena::Create( - arena, Cast(modern_value).NativeValue())); + arena, + common_internal::GetUnknown(Cast(modern_value)))); case ValueKind::kType: return CelValue::CreateCelType( CelValue::CelTypeHolder(google::protobuf::Arena::Create( @@ -1185,7 +1187,7 @@ absl::StatusOr FromLegacyValue(google::protobuf::Arena* arena, case CelValue::Type::kMap: return MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); case CelValue::Type::kUnknownSet: - return UnknownValue{*legacy_value.UnknownSetOrDie()}; + return common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie()); case CelValue::Type::kCelType: return CreateTypeValueFromView(arena, legacy_value.CelTypeOrDie().value()); @@ -1236,7 +1238,7 @@ absl::StatusOr ToLegacyValue( return common_internal::LegacyTrivialMapValue(arena, value); case ValueKind::kUnknown: return CelValue::CreateUnknownSet(google::protobuf::Arena::Create( - arena, Cast(value).NativeValue())); + arena, common_internal::GetUnknown(Cast(value)))); case ValueKind::kType: return CelValue::CreateCelType( CelValue::CelTypeHolder(google::protobuf::Arena::Create( diff --git a/common/values/unknown_value.h b/common/values/unknown_value.h index 9e8ddaae0..1d17de1ca 100644 --- a/common/values/unknown_value.h +++ b/common/values/unknown_value.h @@ -40,13 +40,21 @@ namespace cel { class Value; class UnknownValue; +namespace common_internal { +[[nodiscard]] +UnknownValue MakeUnknownValue(Unknown value); +[[nodiscard]] +Unknown GetUnknown(const UnknownValue& value); +[[nodiscard]] +const FunctionResultSet& GetUnknownFunctionResultSet( + const UnknownValue& value ABSL_ATTRIBUTE_LIFETIME_BOUND); +} // namespace common_internal + // `UnknownValue` represents values of the primitive `duration` type. class UnknownValue final : private common_internal::ValueMixin { public: static constexpr ValueKind kKind = ValueKind::kUnknown; - explicit UnknownValue(Unknown unknown) : unknown_(std::move(unknown)) {} - UnknownValue() = default; UnknownValue(const UnknownValue&) = default; UnknownValue(UnknownValue&&) = default; @@ -80,31 +88,25 @@ class UnknownValue final : private common_internal::ValueMixin { bool IsZeroValue() const { return false; } - void swap(UnknownValue& other) noexcept { - using std::swap; - swap(unknown_, other.unknown_); - } - - const Unknown& NativeValue() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { - return unknown_; - } - - Unknown NativeValue() && { - Unknown unknown = std::move(unknown_); - return unknown; - } - - const AttributeSet& attribute_set() const { + [[nodiscard]] + AttributeSet ToAttributeSet() const { return unknown_.unknown_attributes(); } - const FunctionResultSet& function_result_set() const { - return unknown_.unknown_function_results(); + void swap(UnknownValue& other) noexcept { + using std::swap; + swap(unknown_, other.unknown_); } private: + friend UnknownValue common_internal::MakeUnknownValue(Unknown value); + friend Unknown common_internal::GetUnknown(const UnknownValue&); + friend const FunctionResultSet& common_internal::GetUnknownFunctionResultSet( + const UnknownValue& value); friend class common_internal::ValueMixin; + explicit UnknownValue(Unknown unknown) : unknown_(std::move(unknown)) {} + Unknown unknown_; }; @@ -116,6 +118,26 @@ inline std::ostream& operator<<(std::ostream& out, const UnknownValue& value) { return out << value.DebugString(); } +namespace common_internal { + +[[nodiscard]] +inline UnknownValue MakeUnknownValue(Unknown value) { + return UnknownValue(std::move(value)); +} + +[[nodiscard]] +inline Unknown GetUnknown(const UnknownValue& value) { + return value.unknown_; +} + +[[nodiscard]] +inline const FunctionResultSet& GetUnknownFunctionResultSet( + const UnknownValue& value ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return value.unknown_.unknown_function_results(); +} + +} // namespace common_internal + } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_UNKNOWN_VALUE_H_ diff --git a/eval/eval/attribute_utility.cc b/eval/eval/attribute_utility.cc index 1e044627e..cea676efd 100644 --- a/eval/eval/attribute_utility.cc +++ b/eval/eval/attribute_utility.cc @@ -118,16 +118,19 @@ absl::optional AttributeUtility::MergeUnknowns( const auto& current_set = value.GetUnknown(); cel::base_internal::UnknownSetAccess::Add( - *result_set, UnknownSet(current_set.attribute_set(), - current_set.function_result_set())); + *result_set, + UnknownSet( + current_set.ToAttributeSet(), + cel::common_internal::GetUnknownFunctionResultSet(current_set))); } if (!result_set.has_value()) { return std::nullopt; } - return UnknownValue(cel::Unknown(result_set->unknown_attributes(), - result_set->unknown_function_results())); + return cel::common_internal::MakeUnknownValue( + cel::Unknown(result_set->unknown_attributes(), + result_set->unknown_function_results())); } UnknownValue AttributeUtility::MergeUnknownValues( @@ -136,12 +139,13 @@ UnknownValue AttributeUtility::MergeUnknownValues( // distinguish unset (nullopt) and empty(engaged empty value). AttributeSet attributes; FunctionResultSet function_results; - attributes.Add(left.attribute_set()); - function_results.Add(left.function_result_set()); - attributes.Add(right.attribute_set()); - function_results.Add(right.function_result_set()); + attributes.Add(left.ToAttributeSet()); + function_results.Add(cel::common_internal::GetUnknownFunctionResultSet(left)); + attributes.Add(right.ToAttributeSet()); + function_results.Add( + cel::common_internal::GetUnknownFunctionResultSet(right)); - return UnknownValue( + return cel::common_internal::MakeUnknownValue( cel::Unknown(std::move(attributes), std::move(function_results))); } @@ -191,16 +195,17 @@ absl::optional AttributeUtility::IdentifyAndMergeUnknowns( if (arg_unknowns.has_value()) { cel::base_internal::UnknownSetAccess::Add( - *result_set, UnknownSet((*arg_unknowns).attribute_set(), - (*arg_unknowns).function_result_set())); + *result_set, cel::common_internal::GetUnknown(*arg_unknowns)); } - return UnknownValue(cel::Unknown(result_set->unknown_attributes(), - result_set->unknown_function_results())); + return cel::common_internal::MakeUnknownValue( + cel::Unknown(result_set->unknown_attributes(), + result_set->unknown_function_results())); } UnknownValue AttributeUtility::CreateUnknownSet(cel::Attribute attr) const { - return UnknownValue(cel::Unknown(AttributeSet({std::move(attr)}))); + return cel::common_internal::MakeUnknownValue( + cel::Unknown(AttributeSet({std::move(attr)}))); } absl::StatusOr AttributeUtility::CreateMissingAttributeError( @@ -213,13 +218,14 @@ absl::StatusOr AttributeUtility::CreateMissingAttributeError( UnknownValue AttributeUtility::CreateUnknownSet( const cel::FunctionDescriptor& fn_descriptor, int64_t expr_id, absl::Span args) const { - return UnknownValue( + return cel::common_internal::MakeUnknownValue( cel::Unknown(FunctionResultSet(FunctionResult(fn_descriptor, expr_id)))); } void AttributeUtility::Add(Accumulator& a, const cel::UnknownValue& v) const { - a.attribute_set_.Add(v.attribute_set()); - a.function_result_set_.Add(v.function_result_set()); + a.attribute_set_.Add(v.ToAttributeSet()); + a.function_result_set_.Add( + cel::common_internal::GetUnknownFunctionResultSet(v)); } void AttributeUtility::Add(Accumulator& a, const AttributeTrail& attr) const { @@ -253,7 +259,7 @@ bool Accumulator::IsEmpty() const { } cel::UnknownValue Accumulator::Build() && { - return cel::UnknownValue( + return cel::common_internal::MakeUnknownValue( cel::Unknown(std::move(attribute_set_), std::move(function_result_set_))); } diff --git a/eval/eval/attribute_utility_test.cc b/eval/eval/attribute_utility_test.cc index f3dbc0d06..ea42c5b3b 100644 --- a/eval/eval/attribute_utility_test.cc +++ b/eval/eval/attribute_utility_test.cc @@ -85,10 +85,10 @@ TEST_F(AttributeUtilityTest, UnknownsUtilityMergeUnknownsFromValues) { AttributeUtility utility(unknown_patterns, missing_attribute_patterns); - UnknownValue unknown_set0 = - cel::UnknownValue(cel::Unknown(AttributeSet({attribute0}))); - UnknownValue unknown_set1 = - cel::UnknownValue(cel::Unknown(AttributeSet({attribute1}))); + UnknownValue unknown_set0 = cel::common_internal::MakeUnknownValue( + cel::Unknown(AttributeSet({attribute0}))); + UnknownValue unknown_set1 = cel::common_internal::MakeUnknownValue( + cel::Unknown(AttributeSet({attribute1}))); std::vector values = { unknown_set0, @@ -99,7 +99,7 @@ TEST_F(AttributeUtilityTest, UnknownsUtilityMergeUnknownsFromValues) { absl::optional unknown_set = utility.MergeUnknowns(values); ASSERT_TRUE(unknown_set.has_value()); - EXPECT_THAT((*unknown_set).attribute_set(), + EXPECT_THAT((*unknown_set).ToAttributeSet(), UnorderedPointwise( Eq(), std::vector{attribute0, attribute1})); } @@ -162,8 +162,8 @@ TEST_F(AttributeUtilityTest, CreateUnknownSet) { AttributeUtility utility(empty_patterns, empty_patterns); UnknownValue set = utility.CreateUnknownSet(trail.attribute()); - ASSERT_THAT(set.attribute_set(), SizeIs(1)); - ASSERT_OK_AND_ASSIGN(auto elem, set.attribute_set().begin()->AsString()); + ASSERT_THAT(set.ToAttributeSet(), SizeIs(1)); + ASSERT_OK_AND_ASSIGN(auto elem, set.ToAttributeSet().begin()->AsString()); EXPECT_EQ(elem, "destination.ip"); } diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index 990003823..dfc9557a6 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -329,9 +329,9 @@ TEST(CreateDirectListStep, ForwardFirstError) { std::vector UnknownAttrNames(const UnknownValue& v) { std::vector names; - names.reserve(v.attribute_set().size()); + names.reserve(v.ToAttributeSet().size()); - for (const auto& attr : v.attribute_set()) { + for (const auto& attr : v.ToAttributeSet()) { EXPECT_OK(attr.AsString().status()); names.push_back(attr.AsString().value_or("")); } @@ -355,10 +355,14 @@ TEST(CreateDirectListStep, MergeUnknowns) { AttributeSet attr_set2({Attribute("var2")}); std::vector> deps; - deps.push_back(CreateConstValueDirectStep( - cel::UnknownValue(cel::Unknown(std::move(attr_set1))), -1)); - deps.push_back(CreateConstValueDirectStep( - cel::UnknownValue(cel::Unknown(std::move(attr_set2))), -1)); + deps.push_back( + CreateConstValueDirectStep(cel::common_internal::MakeUnknownValue( + cel::Unknown(std::move(attr_set1))), + -1)); + deps.push_back( + CreateConstValueDirectStep(cel::common_internal::MakeUnknownValue( + cel::Unknown(std::move(attr_set2))), + -1)); auto step = CreateDirectListStep(std::move(deps), {}, -1); cel::Value result; diff --git a/eval/eval/ident_step_test.cc b/eval/eval/ident_step_test.cc index ce10d7d98..da5c0dc84 100644 --- a/eval/eval/ident_step_test.cc +++ b/eval/eval/ident_step_test.cc @@ -259,7 +259,7 @@ TEST(DirectIdentStepTest, UnknownAttribute) { ASSERT_OK(step->Evaluate(frame, result, trail)); ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).attribute_set(), SizeIs(1)); + EXPECT_THAT(Cast(result).ToAttributeSet(), SizeIs(1)); } TEST(DirectIdentStepTest, MissingAttribute) { diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index 2bdcbc8ea..29725377a 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -356,7 +356,8 @@ struct BinaryTestCase { UnknownValue MakeUnknownValue(std::string attr) { std::vector attrs; attrs.push_back(Attribute(std::move(attr))); - return cel::UnknownValue(cel::Unknown(AttributeSet(attrs))); + return cel::common_internal::MakeUnknownValue( + cel::Unknown(AttributeSet(attrs))); } std::unique_ptr MakeArgStep(OpArg arg, diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index 92cda2fbe..f19370cdc 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -1116,7 +1116,7 @@ class DirectSelectStepTest : public testing::Test { std::vector AttributeStrings(const UnknownValue& v) { std::vector result; - for (const Attribute& attr : v.attribute_set()) { + for (const Attribute& attr : v.ToAttributeSet()) { auto attr_str = attr.AsString(); ABSL_DCHECK_OK(attr_str.status()); result.push_back(std::move(attr_str).value()); @@ -1597,8 +1597,9 @@ TEST_F(DirectSelectStepTest, ForwardUnknownOperand) { AttributeSet attr_set({Attribute("attr", {AttributeQualifier::OfInt(0)})}); auto step = CreateDirectSelectStep( - CreateConstValueDirectStep( - cel::UnknownValue(cel::Unknown(std::move(attr_set))), -1), + CreateConstValueDirectStep(cel::common_internal::MakeUnknownValue( + cel::Unknown(std::move(attr_set))), + -1), cel::StringValue("single_int64"), /*test_only=*/false, -1, /*enable_wrapper_type_null_unboxing=*/true); diff --git a/eval/eval/ternary_step_test.cc b/eval/eval/ternary_step_test.cc index ff66c0308..2c400b3e1 100644 --- a/eval/eval/ternary_step_test.cc +++ b/eval/eval/ternary_step_test.cc @@ -283,8 +283,8 @@ TEST_P(TernaryStepDirectTest, ForwardUnknown) { std::vector attrs{{cel::Attribute("var")}}; - cel::UnknownValue unknown_value = - cel::UnknownValue(cel::Unknown(cel::AttributeSet(attrs))); + cel::UnknownValue unknown_value = cel::common_internal::MakeUnknownValue( + cel::Unknown(cel::AttributeSet(attrs))); std::unique_ptr step = CreateDirectTernaryStep( CreateConstValueDirectStep(unknown_value, -1), @@ -296,7 +296,7 @@ TEST_P(TernaryStepDirectTest, ForwardUnknown) { ASSERT_OK(step->Evaluate(frame, result, attr_unused)); ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue().unknown_attributes(), + EXPECT_THAT(Cast(result).ToAttributeSet(), ElementsAre(Truly([](const cel::Attribute& attr) { return attr.variable_name() == "var"; }))); diff --git a/extensions/select_optimization_test.cc b/extensions/select_optimization_test.cc index 27c191738..d75afe52e 100644 --- a/extensions/select_optimization_test.cc +++ b/extensions/select_optimization_test.cc @@ -1278,7 +1278,7 @@ INSTANTIATE_TEST_SUITE_P( ASSERT_OK_AND_ASSIGN(Value result, got); ASSERT_TRUE(result->Is()) << result->DebugString(); EXPECT_THAT( - result.GetUnknown().attribute_set(), + result.GetUnknown().ToAttributeSet(), ElementsAre(Eq(Attribute( "b", { AttributeQualifier::OfString("child"), @@ -1328,7 +1328,7 @@ INSTANTIATE_TEST_SUITE_P( [](const absl::StatusOr& got) { ASSERT_OK_AND_ASSIGN(Value result, got); ASSERT_TRUE(result->Is()) << result->DebugString(); - EXPECT_THAT(result.GetUnknown().attribute_set(), + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), ElementsAre(Truly([](const Attribute& attr) { return attr.variable_name() == "b"; }))); From 14fad34ddb74f3ca9407539a091df2225147cf4d Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 21 Sep 2026 13:37:29 -0700 Subject: [PATCH 229/350] Add `cel::StringValue::Literal` to create values from compile time constants PiperOrigin-RevId: 985481254 --- common/values/string_value.h | 36 ++++++++++++++++++++++++++++++ common/values/string_value_test.cc | 7 ++++++ 2 files changed, 43 insertions(+) diff --git a/common/values/string_value.h b/common/values/string_value.h index 8045e4b3f..26cdec6c1 100644 --- a/common/values/string_value.h +++ b/common/values/string_value.h @@ -26,6 +26,7 @@ #include #include "absl/base/attributes.h" +#include "absl/base/macros.h" #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" @@ -45,6 +46,16 @@ #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/message.h" +#if ABSL_HAVE_ATTRIBUTE(enable_if) +#define CEL_COMMON_INTERNAL_REQUIRE_CONSTEXPR_STRING_VIEW(array) \ + __attribute__(( \ + enable_if(::cel::common_internal::IsWellFormedStringValueLiteral(array), \ + "Argument '" #array "' must be a 'constexpr char[N]' or a " \ + "'constexpr absl::string_view'."))) +#else +#define CEL_COMMON_INTERNAL_REQUIRE_CONSTEXPR_STRING_VIEW(array) +#endif + namespace cel { class Value; @@ -54,6 +65,14 @@ class StringValue; namespace common_internal { absl::string_view LegacyStringValue(const StringValue& value, bool stable, google::protobuf::Arena* absl_nonnull arena); +constexpr bool IsWellFormedStringValueLiteral(absl::string_view string) { + for (char c : string) { + if (c == '\0') { + return false; + } + } + return true; +} } // namespace common_internal // `StringValue` represents values of the primitive `string` type. @@ -86,6 +105,23 @@ class StringValue final : private common_internal::ValueMixin { // the provided string outlives the use of the returned StringValue. static StringValue WrapUnsafe(absl::string_view value); + // Returns a StringValue that aliases the provided string, which must be a + // compile time constant. This is enforced during compilation when using + // Clang. + template + static StringValue Literal(const char (&value)[N]) + CEL_COMMON_INTERNAL_REQUIRE_CONSTEXPR_STRING_VIEW(value) { + static_assert(N > 0); + ABSL_DCHECK_EQ(value[N - 1], '\0'); + return WrapUnsafe(value); + } + static StringValue Literal(absl::string_view value) + CEL_COMMON_INTERNAL_REQUIRE_CONSTEXPR_STRING_VIEW(value) { + return WrapUnsafe(value); + } + static StringValue Literal(const std::string&) = delete; + static StringValue Literal(std::string&&) = delete; + static StringValue Concat(const StringValue& lhs, const StringValue& rhs, google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND); diff --git a/common/values/string_value_test.cc b/common/values/string_value_test.cc index b4fa404ae..95d72fc65 100644 --- a/common/values/string_value_test.cc +++ b/common/values/string_value_test.cc @@ -518,5 +518,12 @@ TEST_F(StringValueTest, Reverse) { StringValueIs("!enilni derots eb ton ot hguone egral si gnirts sihT")); } +TEST_F(StringValueTest, Literal) { + using ::cel::test::StringValueIs; + + EXPECT_THAT(StringValue::Literal("Hello, World!"), + StringValueIs("Hello, World!")); +} + } // namespace } // namespace cel From a7dedca17b6f9df1d85c12e03acd29368ece895a Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 23 Sep 2026 08:35:08 -0700 Subject: [PATCH 230/350] Enforce recursion depth limits for ternary expressions in the Pratt parser. PiperOrigin-RevId: 986779612 --- parser/internal/pratt_parser_test.cc | 9 +++++++++ parser/internal/pratt_parser_worker.h | 19 +++++++++++-------- parser/parser_test.cc | 6 ++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index fc6477745..0a1b251f5 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -1742,6 +1742,15 @@ TEST(PrattParserRecursionDepthTest, SequentialScopesDoNotAccumulateDepth) { EXPECT_THAT(Parse("[1] + [2] + [3]", options), IsOkAndHolds(NotNull())); } +TEST(PrattParserRecursionDepthTest, DeeplyNestedTernary) { + cel::ParserOptions options; + options.max_recursion_depth = 4; + EXPECT_THAT(Parse("a ? b : a ? b : a ? b : a ? b : c", options), + IsOkAndHolds(NotNull())); + EXPECT_THAT(Parse("a ? b : a ? b : a ? b : a ? b : a ? b : c", options), + StatusIs(absl::StatusCode::kCancelled)); +} + class TestParserWorker : public ParserWorker { // Expose the protected constructor and methods for testing. public: diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 2cfe84b79..f6d0fc825 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -358,18 +358,21 @@ ExprNode PrattParserWorker::ParseExpr() { template void PrattParserWorker::ParseTernary(ExprNode& lhs) { - Token op_tok = NextToken(); - int64_t op_id = NextId(op_tok); - ExprNode true_expr = ParseBinaryAndTernary(1); - if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { + if (recursion_depth_ > options_.max_recursion_depth) { + recursion_limit_exceeded_ = true; return; } - ExprNode false_expr = ParseBinaryAndTernary(0); + recursion_depth_++; + absl::Cleanup depth_cleanup = [this] { recursion_depth_--; }; + int64_t op_id = NextId(NextToken()); std::vector args; args.reserve(3); args.push_back(std::move(lhs)); - args.push_back(std::move(true_expr)); - args.push_back(std::move(false_expr)); + args.push_back(ParseBinaryAndTernary(1)); + if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { + return; + } + args.push_back(ParseBinaryAndTernary(0)); lhs = ast_factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(args)); } @@ -391,7 +394,7 @@ void PrattParserWorker::BuildBinaryCall(int64_t op_id, template ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { ExprNode lhs = ParseSelectorChain(); - while (true) { + while (!recursion_limit_exceeded_ && !is_recovery_limit_exceeded()) { TokenType tok = peek_token_.type; if (tok == TokenType::kQuestion && min_prec <= 0) { ParseTernary(lhs); diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 922968d3e..399ea1403 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1086,6 +1086,12 @@ std::vector test_cases = { "]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]" "]]]]]]", "", "Expression recursion limit exceeded. limit: 32", "", "", ""}, + {"a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : " + "a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : " + "a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : " + "a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : " + "a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : a ? b : c", + "", "Expression recursion limit exceeded. limit: 32", "", "", ""}, { // Note, the ANTLR parse stack may recurse much more deeply and permit // more detailed expressions than the visitor can recurse over in From e83a8d89f33a3a643a9fc0c10c331ff083b76d5b Mon Sep 17 00:00:00 2001 From: Justin King Date: Wed, 23 Sep 2026 08:40:31 -0700 Subject: [PATCH 231/350] Update unknown `FunctionResultSet` to only hold function names PiperOrigin-RevId: 986782397 --- base/BUILD | 6 ++++- base/function_result.h | 21 ++++++--------- eval/eval/attribute_utility.cc | 2 +- .../unknown_function_result_set_test.cc | 27 +++++++------------ eval/public/unknown_set_test.cc | 2 +- eval/tests/unknowns_end_to_end_test.cc | 2 +- 6 files changed, 25 insertions(+), 35 deletions(-) diff --git a/base/BUILD b/base/BUILD index 8c2a32671..600889182 100644 --- a/base/BUILD +++ b/base/BUILD @@ -120,7 +120,11 @@ cc_library( hdrs = [ "function_result.h", ], - deps = [":function_descriptor"], + deps = [ + ":function_descriptor", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/strings:string_view", + ], ) cc_library( diff --git a/base/function_result.h b/base/function_result.h index 977ceeb90..b511dd744 100644 --- a/base/function_result.h +++ b/base/function_result.h @@ -16,8 +16,11 @@ #define THIRD_PARTY_CEL_CPP_BASE_FUNCTION_RESULT_H_ #include +#include #include +#include "absl/base/attributes.h" +#include "absl/strings/string_view.h" #include "base/function_descriptor.h" namespace cel { @@ -32,29 +35,21 @@ class FunctionResult final { FunctionResult& operator=(const FunctionResult&) = default; FunctionResult& operator=(FunctionResult&&) = default; - FunctionResult(FunctionDescriptor descriptor, int64_t expr_id) - : descriptor_(std::move(descriptor)), expr_id_(expr_id) {} + explicit FunctionResult(std::string_view name) : name_(name) {} - // The descriptor of the called function that return Unknown. - const FunctionDescriptor& descriptor() const { return descriptor_; } - - // The id of the |Expr| that triggered the function call step. Provided - // informationally -- if two different |Expr|s generate the same unknown call, - // they will be treated as the same unknown function result. - int64_t call_expr_id() const { return expr_id_; } + absl::string_view name() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return name_; } // Equality operator provided for testing. Compatible with set less-than // comparator. // Compares descriptor then arguments elementwise. bool IsEqualTo(const FunctionResult& other) const { - return descriptor() == other.descriptor(); + return name() == other.name(); } // TODO(uncreated-issue/5): re-implement argument capture private: - FunctionDescriptor descriptor_; - int64_t expr_id_; + std::string name_; }; inline bool operator==(const FunctionResult& lhs, const FunctionResult& rhs) { @@ -62,7 +57,7 @@ inline bool operator==(const FunctionResult& lhs, const FunctionResult& rhs) { } inline bool operator<(const FunctionResult& lhs, const FunctionResult& rhs) { - return lhs.descriptor() < rhs.descriptor(); + return lhs.name() < rhs.name(); } } // namespace cel diff --git a/eval/eval/attribute_utility.cc b/eval/eval/attribute_utility.cc index cea676efd..274deb78c 100644 --- a/eval/eval/attribute_utility.cc +++ b/eval/eval/attribute_utility.cc @@ -219,7 +219,7 @@ UnknownValue AttributeUtility::CreateUnknownSet( const cel::FunctionDescriptor& fn_descriptor, int64_t expr_id, absl::Span args) const { return cel::common_internal::MakeUnknownValue( - cel::Unknown(FunctionResultSet(FunctionResult(fn_descriptor, expr_id)))); + cel::Unknown(FunctionResultSet(FunctionResult(fn_descriptor.name())))); } void AttributeUtility::Add(Accumulator& a, const cel::UnknownValue& v) const { diff --git a/eval/public/unknown_function_result_set_test.cc b/eval/public/unknown_function_result_set_test.cc index 745b5b9ff..b629f12fe 100644 --- a/eval/public/unknown_function_result_set_test.cc +++ b/eval/public/unknown_function_result_set_test.cc @@ -38,15 +38,15 @@ CelFunctionDescriptor kTwoInt("TwoInt", false, CelFunctionDescriptor kOneInt("OneInt", false, {CelValue::Type::kInt64}); TEST(UnknownFunctionResult, Equals) { - UnknownFunctionResult call1(kTwoInt, /*expr_id=*/0); + UnknownFunctionResult call1(kTwoInt.name()); - UnknownFunctionResult call2(kTwoInt, /*expr_id=*/0); + UnknownFunctionResult call2(kTwoInt.name()); EXPECT_TRUE(call1.IsEqualTo(call2)); - UnknownFunctionResult call3(kOneInt, /*expr_id=*/0); + UnknownFunctionResult call3(kOneInt.name()); - UnknownFunctionResult call4(kOneInt, /*expr_id=*/0); + UnknownFunctionResult call4(kOneInt.name()); EXPECT_TRUE(call3.IsEqualTo(call4)); @@ -57,25 +57,16 @@ TEST(UnknownFunctionResult, Equals) { } TEST(UnknownFunctionResult, InequalDescriptor) { - UnknownFunctionResult call1(kTwoInt, /*expr_id=*/0); + UnknownFunctionResult call1(kTwoInt.name()); - UnknownFunctionResult call2(kOneInt, /*expr_id=*/0); + UnknownFunctionResult call2(kOneInt.name()); EXPECT_FALSE(call1.IsEqualTo(call2)); - CelFunctionDescriptor one_uint("OneInt", false, {CelValue::Type::kUint64}); - - UnknownFunctionResult call3(kOneInt, /*expr_id=*/0); - - UnknownFunctionResult call4(one_uint, /*expr_id=*/0); - - EXPECT_FALSE(call3.IsEqualTo(call4)); - - UnknownFunctionResultSet call_set({call1, call3, call4}); - EXPECT_EQ(call_set.size(), 3); + UnknownFunctionResultSet call_set({call1, call2}); + EXPECT_EQ(call_set.size(), 2); auto it = call_set.begin(); - EXPECT_EQ(*it++, call3); - EXPECT_EQ(*it++, call4); + EXPECT_EQ(*it++, call2); EXPECT_EQ(*it++, call1); } diff --git a/eval/public/unknown_set_test.cc b/eval/public/unknown_set_test.cc index 3a0d151a5..c15030c00 100644 --- a/eval/public/unknown_set_test.cc +++ b/eval/public/unknown_set_test.cc @@ -22,7 +22,7 @@ using ::testing::UnorderedElementsAre; UnknownFunctionResultSet MakeFunctionResult(Arena* arena, int64_t id) { CelFunctionDescriptor desc("OneInt", false, {CelValue::Type::kInt64}); - return UnknownFunctionResultSet(UnknownFunctionResult(desc, /*expr_id=*/0)); + return UnknownFunctionResultSet(UnknownFunctionResult(desc.name())); } UnknownAttributeSet MakeAttribute(Arena* arena, int64_t id) { diff --git a/eval/tests/unknowns_end_to_end_test.cc b/eval/tests/unknowns_end_to_end_test.cc index 71ffe652c..b8132e42f 100644 --- a/eval/tests/unknowns_end_to_end_test.cc +++ b/eval/tests/unknowns_end_to_end_test.cc @@ -133,7 +133,7 @@ class UnknownsTest : public testing::Test { MATCHER_P(FunctionCallIs, fn_name, "") { const cel::FunctionResult& result = arg; - return result.descriptor().name() == fn_name; + return result.name() == fn_name; } MATCHER_P(AttributeIs, attr, "") { From 2ac4c549776cef92bcbde790f1cd0348e1c3b6c5 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 23 Sep 2026 11:31:14 -0700 Subject: [PATCH 232/350] No public description PiperOrigin-RevId: 986892031 --- eval/eval/logic_step.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index 9e65db8f0..ed0a95275 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -190,9 +190,10 @@ class LogicalOpStep : public ExpressionStepBase { public: // Constructs FunctionStep that uses overloads specified. LogicalOpStep(OpType op_type, size_t count, int64_t expr_id) - : ExpressionStepBase(expr_id), op_type_(op_type), count_(count) { - shortcircuit_ = (op_type_ == OpType::kOr); - } + : ExpressionStepBase(expr_id), + shortcircuit_(op_type == OpType::kOr), + op_type_(op_type), + count_(count) {} absl::Status Evaluate(ExecutionFrame* frame) const override; @@ -247,9 +248,9 @@ class LogicalOpStep : public ExpressionStepBase { } } + bool shortcircuit_; const OpType op_type_; size_t count_; - bool shortcircuit_; }; absl::Status LogicalOpStep::Evaluate(ExecutionFrame* frame) const { From 22168df1198f6014e64110b77a89d9fb357f0ea1 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 23 Sep 2026 11:44:40 -0700 Subject: [PATCH 233/350] No public description PiperOrigin-RevId: 986901003 --- eval/eval/select_step.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 0adb77fd6..ec9e924aa 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -211,10 +211,10 @@ class SelectStep : public ExpressionStepBase { : ExpressionStepBase(expr_id), field_value_(std::move(value)), field_(field_value_.ToString()), - test_field_presence_(test_field_presence), unboxing_option_(enable_wrapper_type_null_unboxing ? ProtoWrapperTypeOptions::kUnsetNull : ProtoWrapperTypeOptions::kUnsetProtoDefault), + test_field_presence_(test_field_presence), enable_optional_types_(enable_optional_types) {} absl::Status Evaluate(ExecutionFrame* frame) const override; @@ -222,8 +222,8 @@ class SelectStep : public ExpressionStepBase { protected: cel::StringValue field_value_; std::string field_; - bool test_field_presence_; ProtoWrapperTypeOptions unboxing_option_; + bool test_field_presence_; bool enable_optional_types_; }; From ff22f6cfbcb62d48b4387501ff29fb197f38e628 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 23 Sep 2026 14:46:19 -0700 Subject: [PATCH 234/350] Iteratively parse nested parentheses in PrattParserWorker Replace the lookahead-based grouping parentheses counter (`CountGroupingParentheses`) with a bottom-up iterative approach in `PrattParserWorker::ParsePrimary()` to eliminate quadratic $O(N^2)$ lookahead scans while keeping $O(1)$ call-stack frames on deeply nested parenthesized expressions. ### Benchmarks (`pratt_parser_benchmark`, `-c opt --dynamic_mode=off`) | Benchmark Case | Before (Mean) | After (Mean) | Change / Speedup | | :--- | ---: | ---: | ---: | | `BM_Pratt_ParseNestedParentheses/10` | 724 ns | 535 ns | -26.1% (1.35x faster) | | `BM_Pratt_ParseNestedParentheses/50` | 2,484 ns | 1,625 ns | -34.6% (1.53x faster) | | `BM_Pratt_ParseNestedParentheses/200` | 8,859 ns | 5,427 ns | -38.7% (1.63x faster) | | `BM_Pratt_ParseNestedLeftParenthesesCalc/10` | 7,876 ns | 3,397 ns | -56.9% (2.32x faster) | | `BM_Pratt_ParseNestedLeftParenthesesCalc/50` | 112,539 ns | 14,518 ns | -87.1% (7.75x faster) | | `BM_Pratt_ParseNestedLeftParenthesesCalc/200` | 1,583,442 ns | 57,375 ns | -96.4% (27.6x faster) | PiperOrigin-RevId: 987017594 --- parser/internal/BUILD | 1 - parser/internal/pratt_parser_benchmark.cc | 39 ++- parser/internal/pratt_parser_test.cc | 39 ++- parser/internal/pratt_parser_worker.cc | 8 +- parser/internal/pratt_parser_worker.h | 285 ++++++++++++-------- parser/parser_test.cc | 304 +++++++++++++++++++++- 6 files changed, 552 insertions(+), 124 deletions(-) diff --git a/parser/internal/BUILD b/parser/internal/BUILD index d679e2853..aca72c689 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -161,7 +161,6 @@ cc_library( "//parser:parser_interface", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/parser/internal/pratt_parser_benchmark.cc b/parser/internal/pratt_parser_benchmark.cc index 952585306..53d451bf1 100644 --- a/parser/internal/pratt_parser_benchmark.cc +++ b/parser/internal/pratt_parser_benchmark.cc @@ -222,9 +222,46 @@ void BM_Antlr_ParseNestedParentheses(benchmark::State& state) { BM_ParseNestedParentheses(state, ParserImplType::kAntlr); } -BENCHMARK(BM_Pratt_ParseNestedParentheses)->Arg(10)->Arg(50); +BENCHMARK(BM_Pratt_ParseNestedParentheses)->Arg(10)->Arg(50)->Arg(200); BENCHMARK(BM_Antlr_ParseNestedParentheses)->Arg(10)->Arg(50); +// ----------------------------------------------------------------------------- +// Workload 5b: Deeply Nested Left Parentheses with Calc ("((((a + 1) + 1))") +// ----------------------------------------------------------------------------- +std::string BuildNestedLeftParenthesesCalc(int depth) { + std::string expr(depth, '('); + absl::StrAppend(&expr, "1 + 2"); + for (int i = 0; i < depth; ++i) { + absl::StrAppend(&expr, ") + 1"); + } + return expr; +} + +void BM_ParseNestedLeftParenthesesCalc(benchmark::State& state, + ParserImplType type) { + cel::ParserOptions options; + auto parser = CreateParser(type, options); + std::string expr = BuildNestedLeftParenthesesCalc(state.range(0)); + + for (auto _ : state) { + auto source = cel::NewSource(expr); + ABSL_DCHECK_OK(source.status()); + auto ast = parser->Parse(**source); + ABSL_DCHECK_OK(ast.status()); + benchmark::DoNotOptimize(ast); + } +} + +void BM_Pratt_ParseNestedLeftParenthesesCalc(benchmark::State& state) { + BM_ParseNestedLeftParenthesesCalc(state, ParserImplType::kPratt); +} +void BM_Antlr_ParseNestedLeftParenthesesCalc(benchmark::State& state) { + BM_ParseNestedLeftParenthesesCalc(state, ParserImplType::kAntlr); +} + +BENCHMARK(BM_Pratt_ParseNestedLeftParenthesesCalc)->Arg(10)->Arg(50)->Arg(200); +BENCHMARK(BM_Antlr_ParseNestedLeftParenthesesCalc)->Arg(10)->Arg(50); + // ----------------------------------------------------------------------------- // Workload 6: Common Representative Expressions with Syntax Errors // ----------------------------------------------------------------------------- diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 0a1b251f5..badc0ffce 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -1698,15 +1698,12 @@ TEST(PrattParserRecursionDepthTest, ParseRecursionDepth) { StatusIs(absl::StatusCode::kCancelled)); } -TEST(PrattParserRecursionDepthTest, ParseRecursionDepthIgnoreExtraParens) { - cel::ParserOptions options; - options.max_recursion_depth = 1; - EXPECT_THAT(Parse("((((1))))", options), IsOkAndHolds(NotNull())); -} - +// Parentheses are unwound iteratively, so nesting them does not grow the C++ +// call stack. The recursion limit is raised above the nesting level here to +// exercise that: each '(' still consumes one unit of the recursion budget. TEST(PrattParserRecursionDepthTest, DeeplyNestedParens) { cel::ParserOptions options; - options.max_recursion_depth = 1; + options.max_recursion_depth = 2100; std::string literal_expr = std::string(1000, '(') + "42" + std::string(1000, ')'); EXPECT_THAT(Parse(literal_expr, options), IsOkAndHolds(NotNull())); @@ -1714,6 +1711,12 @@ TEST(PrattParserRecursionDepthTest, DeeplyNestedParens) { std::string binary_expr = std::string(1000, '(') + "1 + 2" + std::string(1000, ')'); EXPECT_THAT(Parse(binary_expr, options), IsOkAndHolds(NotNull())); + + std::string left_nested_calc_expr = std::string(1000, '(') + "1 + 2"; + for (int i = 0; i < 1000; ++i) { + left_nested_calc_expr += ") + 1"; + } + EXPECT_THAT(Parse(left_nested_calc_expr, options), IsOkAndHolds(NotNull())); } TEST(PrattParserRecursionDepthTest, NestedAndGroupingParensCombinations) { @@ -1724,6 +1727,21 @@ TEST(PrattParserRecursionDepthTest, NestedAndGroupingParensCombinations) { EXPECT_THAT("f((((1))), (((2))))", AstEq("f(1, 2)")); EXPECT_THAT("[{((1)): ((2))}]", AstEq("[{1: 2}]")); EXPECT_THAT("(((a))).b[0]", AstEq("a.b[0]")); + EXPECT_THAT("((((a).b[0]) + 1) ? 2 : 3)", AstEq("(a.b[0] + 1) ? 2 : 3")); + EXPECT_THAT("a ? ((((b)))) : c", AstEq("a ? b : c")); + EXPECT_THAT("((((7) + 1) + 1) + 1)", AstEq("7 + 1 + 1 + 1")); + EXPECT_THAT("(true) || (true || (true || false))", + AstEq("true || (true || (true || false))")); + EXPECT_THAT("(1 + 1 + 1) + 1", AstEq("1 + 1 + 1 + 1")); + EXPECT_THAT("((1 + 1) + 1) + 1", AstEq("1 + 1 + 1 + 1")); + EXPECT_THAT("(((1 + 1 + 1))) + 1", AstEq("1 + 1 + 1 + 1")); + EXPECT_THAT("1 + (1 + 1 + 1)", AstEq("1 + (1 + 1 + 1)")); + EXPECT_THAT("((1 + ((7))))", AstEq("1 + 7")); + EXPECT_THAT("(a.b.c).d.e.f", AstEq("a.b.c.d.e.f")); + EXPECT_THAT("((a && b.c.d).e)", AstEq("(a && b.c.d).e")); + EXPECT_THAT("((a && b && c.d.e).f)", AstEq("(a && b && c.d.e).f")); + EXPECT_THAT("((a ? b : c).d[0] ? (e ? f : g) : h) + ((x).y)", + AstEq("((a ? b : c).d[0] ? (e ? f : g) : h) + x.y")); } TEST(PrattParserRecursionDepthTest, MismatchedParensStillReportErrors) { @@ -1744,13 +1762,18 @@ TEST(PrattParserRecursionDepthTest, SequentialScopesDoNotAccumulateDepth) { TEST(PrattParserRecursionDepthTest, DeeplyNestedTernary) { cel::ParserOptions options; - options.max_recursion_depth = 4; + options.max_recursion_depth = 5; EXPECT_THAT(Parse("a ? b : a ? b : a ? b : a ? b : c", options), IsOkAndHolds(NotNull())); EXPECT_THAT(Parse("a ? b : a ? b : a ? b : a ? b : a ? b : c", options), StatusIs(absl::StatusCode::kCancelled)); } +TEST(PrattParserRecursionDepthTest, NestedParensWithTernaryAndSelectors) { + EXPECT_THAT(Parse("((a ? b : c).d[0] ? (e ? f : g) : h) + ((x).y)"), + IsOkAndHolds(NotNull())); +} + class TestParserWorker : public ParserWorker { // Expose the protected constructor and methods for testing. public: diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index b01408774..1a086c421 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -123,7 +123,7 @@ std::string ParserWorker::GetTokenText(const Token& tok) const { return ""; } -Token ParserWorker::NextSignificantToken(bool report_error) { +Token ParserWorker::NextSignificantToken() { if (is_recovery_limit_exceeded()) { return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; } @@ -132,7 +132,7 @@ Token ParserWorker::NextSignificantToken(bool report_error) { if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) { continue; } - if (tok.type == TokenType::kError && report_error) { + if (tok.type == TokenType::kError) { ReportSyntaxError(tok, lexer_.GetError().message); if (is_recovery_limit_exceeded()) { return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; @@ -159,7 +159,7 @@ bool ParserWorker::Expect(TokenType type, absl::string_view msg) { NextToken(); return true; } - if (is_recovery_limit_exceeded()) { + if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { return false; } if (peek_token_.type != TokenType::kError) { @@ -184,7 +184,7 @@ bool ParserWorker::Expect(TokenType type, absl::string_view msg) { } void ParserWorker::SynchronizeOnDelimiter() { - if (is_recovery_limit_exceeded()) { + if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; return; } diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index f6d0fc825..80a199989 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -25,10 +25,8 @@ #include #include -#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" -#include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_map.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" @@ -74,7 +72,7 @@ class ParserWorker { const cel::ParserOptions& options() const { return options_; } // Token stream management void InitTokenStream(); - Token NextSignificantToken(bool report_error = true); + Token NextSignificantToken(); Token NextToken(); bool Expect(TokenType type, absl::string_view msg = ""); std::string GetTokenText(const Token& tok) const; @@ -100,6 +98,27 @@ class ParserWorker { } } + // Recursion budget accounting. + // + // `recursion_depth_` tracks the levels currently held by live parse frames. + // Chains that are parsed iteratively (operator chains, selector chains, runs + // of parentheses or unary operators) do not add frames, so they report the + // levels they consume through `chain_depth` arguments and through + // `last_parsed_depth_` instead. Counting them keeps `max_recursion_depth` a + // bound on the depth of the resulting AST, as it is for the ANTLR parser. + // + // Returns true and trips the recursion limit if parsing `chain_depth` further + // levels on top of the current depth would exceed the configured limit. + bool CheckRecursion(int chain_depth) { + if (ABSL_PREDICT_FALSE(recursion_depth_ + chain_depth > + options_.max_recursion_depth)) { + recursion_limit_exceeded_ = true; + peek_token_ = Token{.type = TokenType::kEnd, .start = 0, .end = 0}; + return true; + } + return false; + } + // Error reporting and recovery bool is_recovery_limit_exceeded() const { return error_count_ > options_.error_recovery_limit; @@ -117,6 +136,9 @@ class ParserWorker { Token current_token_; Token peek_token_; int recursion_depth_ = 0; + // Depth of the subtree parsed most recently, reported upward so that callers + // can fold it into their own chain depth. + int last_parsed_depth_ = 0; int64_t next_id_ = 1; bool node_limit_exceeded_ = false; absl::flat_hash_map positions_; @@ -208,6 +230,12 @@ class PrattParserWorker : public ParserWorker { using CelOperator = ::google::api::expr::common::CelOperator; ExprNode ParseExpr(); + + // Parses one element of a delimited construct (list, map, struct or argument + // list), accumulating `last_parsed_depth_` to the deepest element seen so + // far. A construct is as deep as its deepest element, not its last one, so + // callers must reset `last_parsed_depth_` to 0 before the first element. + ExprNode ParseElementExpr(); // Parses binary operator expressions and ternary conditional expressions // (`? :`) using operator-precedence (Pratt) parsing. Consumes operators from // the token stream whose binding precedence is greater than or equal to @@ -221,6 +249,10 @@ class PrattParserWorker : public ParserWorker { // `?`, consumes `?`, and recurses with `ParseBinary(1)` for true branch `b` // and `ParseBinary(0)` for false branch `c`. ExprNode ParseBinaryAndTernary(int min_prec); + // `initial_chain_depth` is the depth already accumulated by `lhs`, which the + // operators consumed by this loop keep accumulating on top of. + void ParseBinaryAndTernaryFromLhs(ExprNode& lhs, int min_prec, + int initial_chain_depth); // Parses ternary conditional expressions (`condition ? true_expr : // false_expr`). @@ -253,7 +285,11 @@ class PrattParserWorker : public ParserWorker { // `Type{field: val}`). // // Processes continuous postfix operation chains iteratively. - void ParseSelectorChainTail(ExprNode& lhs); + // + // `initial_chain_depth` is the depth already accumulated by `lhs`: a + // parenthesized primary such as `(a.b.c)` contributes its own depth, which + // the selectors trailing the closing `)` continue to accumulate on top of. + void ParseSelectorChainTail(ExprNode& lhs, int initial_chain_depth); // Parses prefix unary operators (logical NOT `!` and negation `-`). If a // numeric literal immediately follows `-`, folds it directly into a negative @@ -322,8 +358,6 @@ class PrattParserWorker : public ParserWorker { std::optional target, std::vector arguments); - int CountGroupingParentheses(); - AstFactoryInterface& ast_factory_; absl::flat_hash_map macro_calls_; }; @@ -346,8 +380,7 @@ ExprNode PrattParserWorker::ParseExpr() { if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { return ExprNode(); } - if (recursion_depth_ > options_.max_recursion_depth) { - recursion_limit_exceeded_ = true; + if (CheckRecursion(1)) { return ExprNode(); } recursion_depth_++; @@ -356,14 +389,16 @@ ExprNode PrattParserWorker::ParseExpr() { return expr; } +template +ExprNode PrattParserWorker::ParseElementExpr() { + int max_depth = last_parsed_depth_; + ExprNode expr = ParseExpr(); + last_parsed_depth_ = std::max(max_depth, last_parsed_depth_); + return expr; +} + template void PrattParserWorker::ParseTernary(ExprNode& lhs) { - if (recursion_depth_ > options_.max_recursion_depth) { - recursion_limit_exceeded_ = true; - return; - } - recursion_depth_++; - absl::Cleanup depth_cleanup = [this] { recursion_depth_--; }; int64_t op_id = NextId(NextToken()); std::vector args; args.reserve(3); @@ -372,7 +407,7 @@ void PrattParserWorker::ParseTernary(ExprNode& lhs) { if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { return; } - args.push_back(ParseBinaryAndTernary(0)); + args.push_back(ParseExpr()); lhs = ast_factory_.NewCall(op_id, CelOperator::CONDITIONAL, std::move(args)); } @@ -394,6 +429,14 @@ void PrattParserWorker::BuildBinaryCall(int64_t op_id, template ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { ExprNode lhs = ParseSelectorChain(); + ParseBinaryAndTernaryFromLhs(lhs, min_prec, last_parsed_depth_); + return lhs; +} + +template +void PrattParserWorker::ParseBinaryAndTernaryFromLhs( + ExprNode& lhs, int min_prec, int initial_chain_depth) { + int chain_depth = initial_chain_depth; while (!recursion_limit_exceeded_ && !is_recovery_limit_exceeded()) { TokenType tok = peek_token_.type; if (tok == TokenType::kQuestion && min_prec <= 0) { @@ -410,11 +453,20 @@ ExprNode PrattParserWorker::ParseBinaryAndTernary(int min_prec) { } Token op_tok = NextToken(); + if (CheckRecursion(chain_depth)) { + return; + } + chain_depth++; int64_t op_id = NextId(op_tok); BuildBinaryCall(op_id, op_info.name, lhs, ParseBinaryAndTernary(op_info.precedence + 1)); + // `last_parsed_depth_` is the depth of the rhs just parsed. It hangs one + // level below this operator, while `chain_depth` already covers the lhs, so + // the operator node is as deep as whichever side is deeper: `x + a.b.c.d` + // reaches 4 through its rhs and `a.b.c.d + x` reaches 4 through its lhs. + chain_depth = std::max(chain_depth, last_parsed_depth_ + 1); + last_parsed_depth_ = chain_depth; } - return lhs; } // Parses continuous chains of logical operators (`&&`, `||`) iteratively @@ -433,15 +485,17 @@ void PrattParserWorker::ParseBalancedLogicalChain( } lhs = BalanceLogical(op_info.name, std::move(terms), std::move(ops), options_.enable_variadic_logical_operators); + last_parsed_depth_ = 0; } template ExprNode PrattParserWorker::ParseSelectorChain() { + last_parsed_depth_ = 0; ExprNode lhs = ParseUnary(); TokenType tok = peek_token_.type; if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || tok == TokenType::kLeftBrace) { - ParseSelectorChainTail(lhs); + ParseSelectorChainTail(lhs, last_parsed_depth_); } return lhs; } @@ -449,10 +503,16 @@ ExprNode PrattParserWorker::ParseSelectorChain() { // Parses prefix and postfix member/indexing operations iteratively // (e.g., `!a.b[0].c(x)`). template -void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { +void PrattParserWorker::ParseSelectorChainTail( + ExprNode& lhs, int initial_chain_depth) { + int chain_depth = initial_chain_depth; while (true) { TokenType tok = peek_token_.type; if (tok == TokenType::kDot) { + if (CheckRecursion(chain_depth)) { + return; + } + chain_depth++; Token dot_tok = NextToken(); bool optional = false; if (peek_token_.type == TokenType::kQuestion) { @@ -469,6 +529,7 @@ void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { ReportSyntaxError(id_tok, "expected identifier after '.'"); } SynchronizeOnDelimiter(); + last_parsed_depth_ = chain_depth; return; } bool is_member_call = peek_token_.type == TokenType::kLeftParen; @@ -495,10 +556,19 @@ void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { lhs = ast_factory_.NewMemberCall(call_id, id_text, std::move(lhs), std::move(args)); } + // `ParseArguments` leaves `last_parsed_depth_` at the deepest argument. + // Arguments hang one level below the call node, so `a.f(b.c.d.e)` is 4 + // deep. The max preserves the selectors already walked when the + // arguments are shallower, as in `a.b.c.f(1)`. + chain_depth = std::max(chain_depth, last_parsed_depth_ + 1); } else { lhs = ast_factory_.NewSelect(NextId(dot_tok), std::move(lhs), id_text); } } else if (tok == TokenType::kLeftBracket) { + if (CheckRecursion(chain_depth)) { + return; + } + chain_depth++; Token bracket_tok = NextToken(); int64_t op_id = NextId(bracket_tok); bool optional = false; @@ -518,10 +588,22 @@ void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { lhs = ast_factory_.NewCall( op_id, optional ? CelOperator::OPT_INDEX : CelOperator::INDEX, std::move(args)); + // `last_parsed_depth_` is the depth of the index expression just parsed. + // It hangs one level below the index node, so `a[b.c.d.e]` is 4 deep. The + // max preserves the selectors already walked when the index is shallower, + // as in `a.b.c[0]`. + chain_depth = std::max(chain_depth, last_parsed_depth_ + 1); } else if (tok == TokenType::kLeftBrace) { int32_t struct_pos = GetLeftmostPosition(lhs); if (auto struct_name = ExtractStructName(lhs); struct_name.has_value()) { lhs = ParseStruct(NextId(struct_pos), *struct_name); + // `ParseStruct` leaves `last_parsed_depth_` at the deepest field value, + // which carries through unchanged: `Msg{f: a.b.c.d}` is 3 deep. There + // is no `+ 1` here because struct creation is a primary rather than a + // chain link, so it adds no level of its own. The max preserves the + // selectors already walked when the fields are shallower, as in + // `a.b.Msg{f: 1}`. + chain_depth = std::max(chain_depth, last_parsed_depth_); } else { break; } @@ -529,6 +611,7 @@ void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { break; } } + last_parsed_depth_ = chain_depth; } template @@ -568,21 +651,38 @@ ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { op.id = NextId(op.token); } + const bool is_negative_numeric_literal = + has_solitary_trailing_minus && (peek_token_.type == TokenType::kInt || + peek_token_.type == TokenType::kFloat); + int64_t negative_literal_op_id = 0; + if (is_negative_numeric_literal) { + negative_literal_op_id = ops.back().id; + ops.pop_back(); + } + ExprNode operand; + // Every retained operator wraps the operand in one more call node, so the run + // costs as many levels as it has operators even though it is parsed by a + // loop. The outermost one is the deepest, so checking it covers the rest. + if (!ops.empty() && CheckRecursion(static_cast(ops.size()) - 1)) { + return ExprNode(); + } + recursion_depth_ += static_cast(ops.size()); // Match the ANTLR parser behavior where `-(-)+` prefers to match as // repeated negate operators instead of a negation of an int literal. // ---9223372036854775808 will fail to parse. - if (has_solitary_trailing_minus && (peek_token_.type == TokenType::kInt || - peek_token_.type == TokenType::kFloat)) { - int64_t op_id = ops.back().id; - ops.pop_back(); + if (is_negative_numeric_literal) { operand = (peek_token_.type == TokenType::kInt) - ? ParseNegativeIntLiteral(op_id) - : ParseNegativeDoubleLiteral(op_id); - ParseSelectorChainTail(operand); + ? ParseNegativeIntLiteral(negative_literal_op_id) + : ParseNegativeDoubleLiteral(negative_literal_op_id); + ParseSelectorChainTail(operand, /*initial_chain_depth=*/0); } else { operand = ParseSelectorChain(); } + recursion_depth_ -= static_cast(ops.size()); + if (recursion_limit_exceeded_) { + return ExprNode(); + } for (int i = static_cast(ops.size()) - 1; i >= 0; --i) { std::vector args; @@ -626,8 +726,16 @@ ExprNode PrattParserWorker::ParseUnaryOps() { } } + if (CheckRecursion(0)) { + return ExprNode(); + } int64_t op_id = NextId(op); + recursion_depth_++; ExprNode operand = ParseSelectorChain(); + recursion_depth_--; + if (recursion_limit_exceeded_) { + return ExprNode(); + } std::vector args; args.push_back(std::move(operand)); absl::string_view op_name = (op_type == TokenType::kExclamation) @@ -686,14 +794,50 @@ template ExprNode PrattParserWorker::ParsePrimary() { switch (peek_token_.type) { case TokenType::kLeftParen: { - int grouping_paren_count = CountGroupingParentheses(); - for (int i = 0; i < grouping_paren_count; ++i) { + if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { + return ExprNode(); + } + // To avoid deep call-stack recursion on heavily nested parentheses (e.g. + // "((((a))))" or "((((a + 1) + 1) + 1))"), consume all consecutive + // leading '(' tokens upfront, parse the innermost expression once, and + // then iteratively unwind each enclosing '(' from innermost to outermost. + // After consuming each matching ')', if more enclosing '(' remain open + // and the next token is not another ')', continue parsing any trailing + // selectors or binary/ternary operators belonging to that enclosing + // parenthesized level using the already-parsed inner expression as the + // LHS. + int open_parens = 0; + while (peek_token_.type == TokenType::kLeftParen) { + open_parens++; NextToken(); } - ExprNode expr = ParseExpr(); - for (int i = 0; i < grouping_paren_count; ++i) { + // Every '(' is a nesting level and costs one unit of recursion budget, so + // charge all of them here. `recursion_depth_` is already 1 for the + // enclosing expression (as in ParseUnaryOpsChain), so `open_parens` + // parentheses reach depth `recursion_depth_ + open_parens - 1`. + // `recursion_depth_` itself only advances by 1 because the parens are + // unwound iteratively and entering ParseBinaryAndTernary() below adds + // just one stack frame. + if (CheckRecursion(open_parens - 1)) { + return ExprNode(); + } + recursion_depth_++; + ExprNode expr = ParseBinaryAndTernary(0); + int chain_depth = last_parsed_depth_; + for (int i = 0; i < open_parens; ++i) { Expect(TokenType::kRightParen); + if (i < open_parens - 1 && peek_token_.type != TokenType::kRightParen) { + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || + tok == TokenType::kLeftBrace) { + ParseSelectorChainTail(expr, chain_depth); + } + ParseBinaryAndTernaryFromLhs(expr, 0, last_parsed_depth_); + chain_depth = last_parsed_depth_; + } } + recursion_depth_--; + last_parsed_depth_ = chain_depth; return expr; } case TokenType::kNull: @@ -743,6 +887,7 @@ ExprNode PrattParserWorker::ParseList() { Token open_tok = NextToken(); int64_t list_id = NextId(open_tok); ListNodeBuilder builder = ast_factory_.NewListBuilder(list_id); + last_parsed_depth_ = 0; while (peek_token_.type != TokenType::kRightBracket && peek_token_.type != TokenType::kEnd) { bool optional = false; @@ -753,7 +898,7 @@ ExprNode PrattParserWorker::ParseList() { ReportError(q, "unsupported syntax '?'"); } } - builder.Add(ParseExpr(), optional); + builder.Add(ParseElementExpr(), optional); if (peek_token_.type == TokenType::kComma) { NextToken(); } else { @@ -772,6 +917,7 @@ ExprNode PrattParserWorker::ParseMap() { Token open_tok = NextToken(); int64_t map_id = NextId(open_tok); MapNodeBuilder builder = ast_factory_.NewMapBuilder(map_id); + last_parsed_depth_ = 0; while (peek_token_.type != TokenType::kRightBrace && peek_token_.type != TokenType::kEnd) { bool optional = false; @@ -785,13 +931,13 @@ ExprNode PrattParserWorker::ParseMap() { key_start = peek_token_; } int64_t entry_id = NextId(); - ExprNode key = ParseExpr(); + ExprNode key = ParseElementExpr(); Token colon = peek_token_; if (!Expect(TokenType::kColon, "expected ':' in map entry")) { break; } SetPosition(entry_id, colon); - builder.Add(entry_id, std::move(key), ParseExpr(), optional); + builder.Add(entry_id, std::move(key), ParseElementExpr(), optional); if (peek_token_.type == TokenType::kComma) { NextToken(); } else { @@ -811,6 +957,7 @@ ExprNode PrattParserWorker::ParseStruct( Token open_tok = NextToken(); StructNodeBuilder builder = ast_factory_.NewStructBuilder(obj_id, std::string(struct_name)); + last_parsed_depth_ = 0; while (peek_token_.type != TokenType::kRightBrace && peek_token_.type != TokenType::kEnd) { bool optional = false; @@ -834,7 +981,7 @@ ExprNode PrattParserWorker::ParseStruct( break; } int64_t field_id = NextId(colon); - builder.Add(field_id, std::move(field_name), ParseExpr(), optional); + builder.Add(field_id, std::move(field_name), ParseElementExpr(), optional); if (peek_token_.type == TokenType::kComma) { NextToken(); } else { @@ -857,9 +1004,10 @@ template std::vector PrattParserWorker::ParseArguments( TokenType close_token) { std::vector args; + last_parsed_depth_ = 0; if (peek_token_.type != close_token && peek_token_.type != TokenType::kEnd) { while (true) { - args.push_back(ParseExpr()); + args.push_back(ParseElementExpr()); if (peek_token_.type == TokenType::kComma) { NextToken(); if (peek_token_.type == close_token) { @@ -1213,75 +1361,6 @@ void PrattParserWorker::RecordMacroCall( macro_calls_.insert({macro_id, std::move(call_expr)}); } -// Scans ahead in the token stream to detect contiguous grouping -// parentheses (e.g., `((((expr))))`). By determining the number of outermost -// parentheses that enclose the exact same expression and close contiguously, -// the parser unnests them in a single C++ stack frame, avoiding deep recursive -// descent. -template -int PrattParserWorker::CountGroupingParentheses() { - if (peek_token_.type != TokenType::kLeftParen) { - return 0; - } - - // Save lexer position to restore after scanning ahead. - const int32_t saved_pos = lexer_.SavePosition(); - auto restore_lexer = absl::MakeCleanup( - [this, saved_pos] { lexer_.RestorePosition(saved_pos); }); - - int leading_open_parens = 1; - Token tok = this->NextSignificantToken(/*report_error=*/false); - while (tok.type == TokenType::kLeftParen) { - leading_open_parens++; - tok = this->NextSignificantToken(/*report_error=*/false); - } - if (leading_open_parens == 1) { - return 1; - } - - int open_parens = leading_open_parens; - int consecutive_leading_closed = 0; - - while (open_parens > 0) { - if (tok.type == TokenType::kEnd || tok.type == TokenType::kError) { - // Return 1 to ensure the parser consumes '(' and standard error handling - // catches incomplete expressions like `(ident`. - return 1; - } - - if (tok.type == TokenType::kLeftParen) { - // An inner parenthesis opens within the expression - // (e.g. `(x` in `((1 + (x) ))`). - open_parens++; - consecutive_leading_closed = 0; - } else if (tok.type == TokenType::kRightParen) { - if (leading_open_parens == open_parens) { - // All inner parentheses are balanced, so this ')' closes one of the - // initial leading '(' parentheses (e.g. trailing ')' in `(((expr)))`). - leading_open_parens--; - consecutive_leading_closed++; - } else { - // This ')' closes an inner nested parenthesis (e.g. `(1 + 2)` in - // `((1 + 2) * 3)`), not one of the outermost leading parentheses. - consecutive_leading_closed = 0; - } - open_parens--; - } else { - // Non-parenthesis token (identifier, operator, literal, etc.). Any - // preceding ')' did not close the entire expression, so reset the - // contiguous outer closing count. - consecutive_leading_closed = 0; - } - - if (open_parens > 0) { - tok = this->NextSignificantToken(/*report_error=*/false); - } - } - - // Return at least 1 to make sure we catch unclosed expressions like `(ident`. - return std::max(1, consecutive_leading_closed); -} - } // namespace cel::parser_internal #endif // THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 399ea1403..3512e1819 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -50,6 +50,7 @@ using ::cel::ConstantKindCase; using ::cel::ExprKindCase; using ::cel::ExprPrinter; using ::cel::expr::Expr; +using ::testing::AnyOf; using ::testing::HasSubstr; using ::testing::Not; @@ -1985,13 +1986,6 @@ TEST_P(ExpressionImplTest, DisableStandardMacros) { << adorned_string; } -TEST_P(ExpressionImplTest, RecursionDepthIgnoresParentheses) { - options_.max_recursion_depth = options_.enable_pratt_parser ? 2 : 6; - auto result = Parse("(((1 + 2 + 3 + 4 + (5 + 6))))", "", options_); - - EXPECT_THAT(result, IsOk()); -} - class NewParserBuilderTest : public testing::TestWithParam { protected: NewParserBuilderTest() { options_.enable_pratt_parser = GetParam(); } @@ -2320,6 +2314,302 @@ TEST_P(ParserTest, PrepareSourceForwardsCodepointLimit) { EXPECT_EQ(source_default_desc->description(), ""); } +// Test case pinning down the recursion depth that the parsers charge for +// `source`. +// +// `antlr_min_recursion_depth` and `pratt_min_recursion_depth` are the smallest +// `max_recursion_depth` values that still parse `source` with the respective +// parser; parsing with a smaller limit is expected to be rejected. Both parsers +// charge a level for every nesting level of the resulting AST, so the two +// columns only differ by a constant: the ANTLR parser additionally recurses +// through the grammar rules that wrap an expression, while the Pratt parser +// parses operator chains, selector chains and runs of parentheses iteratively +// and charges them against the depth they reach instead. +struct RecursionDepthTestCase { + std::string name; + std::string source; + int antlr_min_recursion_depth; + int pratt_min_recursion_depth; +}; + +std::string Repeat(absl::string_view part, int count) { + std::string result; + for (int i = 0; i < count; ++i) { + absl::StrAppend(&result, part); + } + return result; +} + +std::string WrapInParens(absl::string_view inner, int count) { + return absl::StrCat(Repeat("(", count), inner, Repeat(")", count)); +} + +std::vector GetRecursionDepthTestCases() { + // Nesting level used by the generated expressions below. + constexpr int kDepth = 32; + + return { + // `1 + 2 + ... + 34`: a flat chain of 33 binary operators. + {.name = "LargeCalc", + .source = + [] { + std::vector terms; + terms.reserve(kDepth + 2); + for (int i = 1; i <= kDepth + 2; ++i) { + terms.push_back(absl::StrCat(i)); + } + return absl::StrJoin(terms, " + "); + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // Runs of parentheses are unwound iteratively by the Pratt parser, but + // each one is still a nesting level and is charged as such. + {.name = "NestedParens", + .source = WrapInParens("7", kDepth), + .antlr_min_recursion_depth = kDepth, + .pratt_min_recursion_depth = kDepth}, + {.name = "NestedParensWithCalc", + .source = absl::StrCat(WrapInParens("7", kDepth), " + ", + WrapInParens("7", kDepth - 1)), + .antlr_min_recursion_depth = kDepth, + .pratt_min_recursion_depth = kDepth}, + // `a.f0.f1 ... .f32`: a flat chain of 33 field selections. + {.name = "FieldSelections", + .source = + [] { + std::string source = "a"; + for (int i = 0; i <= kDepth; ++i) { + absl::StrAppend(&source, ".f", i); + } + return source; + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // `a[1][2] ... [33]`: a flat chain of 33 index operations, each of which + // parses its index expression one level deeper. + {.name = "IndexOperations", + .source = + [] { + std::string source = "a"; + for (int i = 1; i <= kDepth + 1; ++i) { + absl::StrAppend(&source, "[", i, "]"); + } + return source; + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // `a < 1 < 2 < ... < 33`: a flat chain of 33 relation operators. + {.name = "RelationOperators", + .source = + [] { + std::string source = "a"; + for (int i = 1; i <= kDepth + 1; ++i) { + absl::StrAppend(&source, " < ", i); + } + return source; + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // 14 operands with 20 index operations each, joined by relation + // operators. + {.name = "IndexRelationOperators", + .source = + [] { + std::string operand = "a"; + for (int i = 1; i <= 20; ++i) { + absl::StrAppend(&operand, "[", i, "]"); + } + return absl::StrJoin(std::vector(14, operand), + " != "); + }(), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 1}, + // `a ? b : a ? b : ... : c`: the false branch of a ternary is parsed + // recursively, so each link costs one level. + {.name = "Ternary", + .source = absl::StrCat(Repeat("a ? b : ", kDepth + 1), "c"), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth + 2}, + {.name = "TernaryTrueBranchParens", + .source = absl::StrCat("a ? ", WrapInParens("b", kDepth), " : c"), + .antlr_min_recursion_depth = kDepth, + .pratt_min_recursion_depth = kDepth}, + // `((((7) + 1) + 1) ... + 1)`: parentheses nested to the left, each level + // adding a binary operator. + {.name = "NestedLeftParensWithCalc", + .source = + absl::StrCat(Repeat("(", kDepth), "7", Repeat(") + 1", kDepth)), + .antlr_min_recursion_depth = kDepth + 1, + .pratt_min_recursion_depth = kDepth}, + // `(true) || (true || ( ... || false) ... )`: parentheses nested to the + // right, so each level is parsed recursively as the right hand side + // operand. + {.name = "NestedRightParensWithLogicalOr", + .source = absl::StrCat("(true) || ", Repeat("(true || ", kDepth), + "false", Repeat(")", kDepth)), + .antlr_min_recursion_depth = kDepth + 2, + .pratt_min_recursion_depth = kDepth}, + // Parens nested inside an operand are charged against the depth reached + // so far rather than accumulating on top of the enclosing parens. + {.name = "GroupingParensAroundCalc", + .source = "((1 + ((7))))", + .antlr_min_recursion_depth = 4, + .pratt_min_recursion_depth = 3}, + {.name = "GroupingParensAroundCalcChain", + .source = "(((1 + 2 + 3 + 4 + (5 + 6))))", + .antlr_min_recursion_depth = 5, + .pratt_min_recursion_depth = 5}, + {.name = "ParenthesizedLhsCalc", + .source = "(1 + 1 + 1) + 1 + 1 + 1", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + {.name = "NestedLeftParensLhsCalc", + .source = "((1 + 1) + 1) + 1 + 1 + 1", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + {.name = "GroupingParensLhsCalc", + .source = "(((1 + 1 + 1))) + 1 + 1 + 1", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + {.name = "ParenthesizedRhsCalc", + .source = "1 + (1 + 1 + 1 + 1 + 1)", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + {.name = "ParenthesizedFieldSelections", + .source = "(a.b.c).d.e.f", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + // Depth accumulated inside a call argument does not carry into the + // enclosing selector and operator chains that wrap the call. + {.name = "CallArgumentFieldSelections", + .source = "f(a.b.c.d).e", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 4}, + {.name = "CallArgumentFieldSelectionsWithCalc", + .source = "x + f(a.b.c.d) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + {.name = "CallArgumentCalcWithCalc", + .source = "x + f(a + b + c + d) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + {.name = "IndexFieldSelectionsWithCalc", + .source = "x + a[b.c.d.e] + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 6}, + {.name = "MemberCallArgumentFieldSelectionsWithCalc", + .source = "x + a.f(b.c.d.e) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 6}, + {.name = "StructFieldSelectionsWithCalc", + .source = "x + Msg{f: a.b.c.d} + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + // A delimited construct contributes its deepest element, not its last + // one. + {.name = "CallArgumentDeepestNotLast", + .source = "x + f(a.b.c.d, 1) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + {.name = "MemberCallArgumentDeepestNotLast", + .source = "x + a.f(b.c.d.e, 1) + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 6}, + {.name = "ListElementDeepestNotLast", + .source = "x + [a.b.c.d, 1][0] + y", + .antlr_min_recursion_depth = 8, + .pratt_min_recursion_depth = 6}, + {.name = "MapValueDeepestNotLast", + .source = "x + {'k': a.b.c.d, 'j': 1}['k'] + y", + .antlr_min_recursion_depth = 8, + .pratt_min_recursion_depth = 6}, + {.name = "MapKeyDeepestNotLast", + .source = "x + {a.b.c.d.e: 1, 'j': 2}['j'] + y", + .antlr_min_recursion_depth = 9, + .pratt_min_recursion_depth = 7}, + {.name = "StructFieldDeepestNotLast", + .source = "x + Msg{f: a.b.c.d, g: 1} + y", + .antlr_min_recursion_depth = 7, + .pratt_min_recursion_depth = 5}, + {.name = "NestedListElementDeepestNotLast", + .source = "x + [[a.b.c.d, 1], 1][0] + y", + .antlr_min_recursion_depth = 9, + .pratt_min_recursion_depth = 6}, + {.name = "ParenthesizedLogicalAndFieldSelection", + .source = "((a && b.c.d).e)", + .antlr_min_recursion_depth = 5, + .pratt_min_recursion_depth = 3}, + {.name = "ParenthesizedLogicalAndChainFieldSelection", + .source = "((a && b && c.d.e).f)", + .antlr_min_recursion_depth = 5, + .pratt_min_recursion_depth = 3}, + {.name = "FieldSelectionsWithCalc", + .source = "a.b.c.d.e + f.g", + .antlr_min_recursion_depth = 6, + .pratt_min_recursion_depth = 5}, + }; +} + +class RecursionDepthTest + : public testing::TestWithParam> { + protected: + RecursionDepthTest() { + options_.enable_pratt_parser = std::get<1>(GetParam()); + } + + const RecursionDepthTestCase& test_case() const { + return std::get<0>(GetParam()); + } + + int min_recursion_depth() const { + return options_.enable_pratt_parser ? test_case().pratt_min_recursion_depth + : test_case().antlr_min_recursion_depth; + } + + ParserOptions options_; +}; + +TEST_P(RecursionDepthTest, AtMinRecursionDepthSucceeds) { + options_.max_recursion_depth = min_recursion_depth(); + + EXPECT_THAT(Parse(test_case().source, "", options_), IsOk()); +} + +TEST_P(RecursionDepthTest, BelowMinRecursionDepthFails) { + if (min_recursion_depth() == 0) { + GTEST_SKIP() << "expression is parsed without recursing"; + } + int limit = min_recursion_depth() - 1; + options_.max_recursion_depth = limit; + + auto result = Parse(test_case().source, "", options_); + + ASSERT_THAT(result, Not(IsOk())); + // The ANTLR parser reports the limit either from the grammar rule listener + // or, if that one does not trip first, from the AST building visitor. + EXPECT_THAT( + result.status().message(), + AnyOf(HasSubstr(absl::StrFormat( + "Expression recursion limit exceeded. limit: %d", limit)), + HasSubstr(absl::StrFormat( + "Exceeded max recursion depth of %d when parsing", limit)))); +} + +std::string RecursionDepthTestName( + const testing::TestParamInfo>& + test_info) { + const RecursionDepthTestCase& test_case = std::get<0>(test_info.param); + bool enable_pratt = std::get<1>(test_info.param); + return absl::StrCat(test_info.index, "_", enable_pratt ? "Pratt_" : "Legacy_", + test_case.name); +} + +INSTANTIATE_TEST_SUITE_P( + CelParserTest, RecursionDepthTest, + testing::Combine(testing::ValuesIn(GetRecursionDepthTestCases()), + TestedParserImpls), + RecursionDepthTestName); + std::string ExpressionTestName( const testing::TestParamInfo>& test_info) { const TestInfo& info = std::get<0>(test_info.param); From 67b1efb00b2f742189667f09e4142f90b4e84bce Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 23 Sep 2026 17:16:12 -0700 Subject: [PATCH 235/350] [Pratt Parser] Address discrepancies between Pratt and ANTLR parsers. Updated `pratt_parser_fuzzer` to check error parity and full `ParsedExpr` equality between the ANTLR and Pratt parsers, and fixed the uncovered discrepancies: 1. **Mixed unary operator chains (e.g., `-!x`, `!-x`, `!-42`, `!-4.2`)**: Reject heterogeneous unary operator chains except when `!` precedes a negative numeric literal (`!-42`, `!-4.2`). 2. **Vertical tab (`\v`, ASCII 11)**: Treat `\v` as an invalid character rather than whitespace. 3. **Chained optional select (`a.?b.?c`) AST positions**: Record the source position of the field name at the start of the root member expression (`a`) rather than the immediate operand. 4. **Numeric literals followed by identifiers (e.g., `9in-x`)**: Allow numeric literals immediately followed by identifier characters so expressions like `9in-x` parse as `9 in -x`. 5. **CEL reserved identifiers in message type names (e.g., `import{}`, `.import{}`, `import.Foo{}`)**: Allow CEL reserved words as message type names in struct creation expressions. Intentional parser differences ignored by `pratt_parser_fuzzer` (where `PrattParser` behavior is preferred): 1. **Raw byte string literal prefixes (`rb'...'`, `rB'...'`, `Rb'...'`, `RB'...'`)**: Pratt accepts both `br` and `rb` prefix orders, whereas ANTLR only accepts `br`. 2. **Standalone commas in empty collection literals (`[,]`, `{,}`, `Msg{,}`)**: Pratt requires at least one element/entry before a trailing comma. 3. **Leading-dot identifier positions (`.R`)**: Pratt records the source position of leading-dot identifiers at the `.` token rather than the identifier token. PiperOrigin-RevId: 987107190 --- parser/internal/BUILD | 1 + parser/internal/lexer.cc | 17 -- parser/internal/lexer_test.cc | 48 ++--- parser/internal/pratt_parser_test.cc | 159 ++++++++++++++- parser/internal/pratt_parser_worker.cc | 35 +++- parser/internal/pratt_parser_worker.h | 266 +++++++++++++------------ parser/parser_test.cc | 150 +++++++++++++- 7 files changed, 498 insertions(+), 178 deletions(-) diff --git a/parser/internal/BUILD b/parser/internal/BUILD index aca72c689..d679e2853 100644 --- a/parser/internal/BUILD +++ b/parser/internal/BUILD @@ -161,6 +161,7 @@ cc_library( "//parser:parser_interface", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc index dfc78e922..66a9f95bb 100644 --- a/parser/internal/lexer.cc +++ b/parser/internal/lexer.cc @@ -27,7 +27,6 @@ #include "absl/functional/function_ref.h" #include "absl/log/absl_check.h" #include "absl/strings/ascii.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" namespace cel::parser_internal { @@ -169,8 +168,6 @@ Token Lexer::Lex() { switch (c) { case '\f': ABSL_FALLTHROUGH_INTENDED; - case '\v': - ABSL_FALLTHROUGH_INTENDED; case '\t': ABSL_FALLTHROUGH_INTENDED; case '\r': @@ -466,8 +463,6 @@ void Lexer::ConsumeWhitespace() { ABSL_FALLTHROUGH_INTENDED; case '\r': ABSL_FALLTHROUGH_INTENDED; - case '\v': - ABSL_FALLTHROUGH_INTENDED; case '\t': Advance(1); break; @@ -638,12 +633,6 @@ Token Lexer::ConsumeNumericLiteral() { "integral literal missing digits after hexadecimal separator"); } auto token_type = ConsumeIntegralSuffix(); - if (ConsumeIf(IsIdentTrailing)) { - return SetError( - start, GetPosition(), - absl::StrCat(TokenTypeToString(token_type), - " literal has unexpected trailing characters")); - } return MakeToken(token_type, start, GetPosition()); } } @@ -667,12 +656,6 @@ Token Lexer::ConsumeNumericLiteral() { } auto token_type = floating_point ? TokenType::kFloat : ConsumeIntegralSuffix(); - if (ConsumeIf(IsIdentTrailing)) { - return SetError( - start, GetPosition(), - absl::StrCat(TokenTypeToString(token_type), - " literal has unexpected trailing characters")); - } return MakeToken(token_type, start, GetPosition()); } diff --git a/parser/internal/lexer_test.cc b/parser/internal/lexer_test.cc index 220c2df48..8625d8b90 100644 --- a/parser/internal/lexer_test.cc +++ b/parser/internal/lexer_test.cc @@ -313,6 +313,24 @@ bytes''' br"raw_bytes" `a.b-c/d e` {TokenType::kWhitespace, " "}, {TokenType::kBytes, "b\"bytes\""}, {TokenType::kRightParen, ")"}}}, + {"NumericFollowedByIdent", + "0x1A_invalid 123_invalid 1x0 2x 9in-x", + {{TokenType::kInt, "0x1A"}, + {TokenType::kIdent, "_invalid"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "123"}, + {TokenType::kIdent, "_invalid"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "1"}, + {TokenType::kIdent, "x0"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "2"}, + {TokenType::kIdent, "x"}, + {TokenType::kWhitespace, " "}, + {TokenType::kInt, "9"}, + {TokenType::kIn, "in"}, + {TokenType::kMinus, "-"}, + {TokenType::kIdent, "x"}}}, }), [](const testing::TestParamInfo& info) { return std::string(info.param.name); @@ -407,32 +425,10 @@ INSTANTIATE_TEST_SUITE_P( "\n | .^", }, LexerErrorTestCase{ - .source = "0x1A_invalid", - .expected_error_message = - "int literal has unexpected trailing characters", - .expected_error_location = "\n | 0x1A_invalid" - "\n | .....^", - }, - LexerErrorTestCase{ - .source = "123_invalid", - .expected_error_message = - "int literal has unexpected trailing characters", - .expected_error_location = "\n | 123_invalid" - "\n | ....^", - }, - LexerErrorTestCase{ - .source = "1x0", - .expected_error_message = - "int literal has unexpected trailing characters", - .expected_error_location = "\n | 1x0" - "\n | ..^", - }, - LexerErrorTestCase{ - .source = "2x", - .expected_error_message = - "int literal has unexpected trailing characters", - .expected_error_location = "\n | 2x" - "\n | ..^", + .source = "1 \v + 2", + .expected_error_message = "unexpected character", + .expected_error_location = "\n | 1 \v + 2" + "\n | ...^", }, LexerErrorTestCase{ .source = "`unterminated quoted", diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index badc0ffce..798bae838 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -503,7 +503,7 @@ std::vector GetParserTestCases() { )", }, TestCase{ - .source = "-!true", + .source = "-(!true)", .expected_ast = R"( -_( !_( @@ -512,6 +512,46 @@ std::vector GetParserTestCases() { )^#1:Expr.Call# )", }, + TestCase{ + .source = "!-42", + .expected_ast = R"( + !_( + -42^#2:int64# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-4.2", + .expected_ast = R"( + !_( + -4.2^#2:double# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "9in-x", + .expected_ast = R"( + @in( + 9^#1:int64#, + -_( + x^#4:Expr.Ident# + )^#3:Expr.Call# + )^#2:Expr.Call# + )", + }, + TestCase{ + .source = "a.?b.?c", + .expected_ast = R"( + _?._( + _?._( + a^#1:Expr.Ident#, + "b"^#3:string# + )^#2:Expr.Call#, + "c"^#5:string# + )^#4:Expr.Call# + )", + .enable_optional_syntax = true, + }, TestCase{ .source = "-.2.V", .expected_ast = R"( @@ -1191,6 +1231,30 @@ std::vector GetParserTestCases() { )", .enable_optional_syntax = true, }, + TestCase{ + .source = "import{}", + .expected_ast = R"( + import{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = ".import{}", + .expected_ast = R"( + .import{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "import.Foo{}", + .expected_ast = R"( + import.Foo{}^#1:Expr.CreateStruct# + )", + }, + TestCase{ + .source = "Foo.import{}", + .expected_ast = R"( + Foo.import{}^#1:Expr.CreateStruct# + )", + }, TestCase{ .source = "(((10 - 3) - 2))", .expected_ast = R"( @@ -1670,6 +1734,88 @@ std::vector GetErrorTestCases() { " | \"😀😀😀😀😀\" ~error\n" " | ........^", }, + ErrorTestCase{ + .source = "-!x", + .expected_error = + "ERROR: :1:2: Syntax error: unexpected token\n" + " | -!x\n" + " | .^\n" + "ERROR: :1:3: Syntax error: unexpected token after " + "expression\n" + " | -!x\n" + " | ..^", + }, + ErrorTestCase{ + .source = "!-x", + .expected_error = "ERROR: :1:2: Syntax error: unexpected '-'\n" + " | !-x\n" + " | .^", + }, + ErrorTestCase{ + .source = "a.in", + .expected_error = + "ERROR: :1:3: Syntax error: expected identifier after " + "'.'\n" + " | a.in\n" + " | ..^", + }, + ErrorTestCase{ + .source = "1 \v + 2", + .expected_error = + "ERROR: :1:3: Syntax error: unexpected character\n" + " | 1 \v + 2\n" + " | ..^", + }, + ErrorTestCase{ + .source = "has(a.`$b`)", + .expected_error = "ERROR: :1:7: unexpected quoted identifier\n" + " | has(a.`$b`)\n" + " | ......^", + .enable_quoted_identifiers = true, + }, + // Parentheses are ignored for AST construction, not message creation + ErrorTestCase{ + .source = "(a){}", + .expected_error = + "ERROR: :1:4: Syntax error: unexpected token after " + "expression\n" + " | (a){}\n" + " | ...^", + }, + ErrorTestCase{ + .source = "(a.b){}", + .expected_error = + "ERROR: :1:6: Syntax error: unexpected token after " + "expression\n" + " | (a.b){}\n" + " | .....^", + }, + ErrorTestCase{ + .source = "(a).b{}", + .expected_error = + "ERROR: :1:6: Syntax error: unexpected token after " + "expression\n" + " | (a).b{}\n" + " | .....^", + }, + // Backtick quoted identifiers should also be rejected on message creation + ErrorTestCase{ + .source = "a.`b-c`{}", + .expected_error = + "ERROR: :1:8: Syntax error: unexpected token after " + "expression\n" + " | a.`b-c`{}\n" + " | .......^", + .enable_quoted_identifiers = true, + }, + // Invalid field names are disallowed for message creation + ErrorTestCase{ + .source = "Msg{`$b`: 1}", + .expected_error = "ERROR: :1:5: unexpected quoted identifier\n" + " | Msg{`$b`: 1}\n" + " | ....^", + .enable_quoted_identifiers = true, + }, }; } @@ -1688,6 +1834,17 @@ TEST(PrattParserTest, SourceInfoPositionsPopulated) { ASSERT_EQ(root.call_expr().args().size(), 2); EXPECT_EQ(source_info.positions().at(root.call_expr().args()[0].id()), 0); EXPECT_EQ(source_info.positions().at(root.call_expr().args()[1].id()), 4); + + cel::ParserOptions opt_options; + opt_options.enable_optional_syntax = true; + ASSERT_OK_AND_ASSIGN(std::unique_ptr opt_ast, + Parse("a.?b.?c", opt_options)); + const auto& opt_pos = opt_ast->source_info().positions(); + EXPECT_EQ(opt_pos.at(1), 0); + EXPECT_EQ(opt_pos.at(2), 1); + EXPECT_EQ(opt_pos.at(3), 0); + EXPECT_EQ(opt_pos.at(4), 4); + EXPECT_EQ(opt_pos.at(5), 0); } TEST(PrattParserRecursionDepthTest, ParseRecursionDepth) { diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index 1a086c421..84d1d6ed6 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -20,6 +20,7 @@ #include "absl/base/nullability.h" #include "absl/base/optimization.h" +#include "absl/cleanup/cleanup.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -123,7 +124,7 @@ std::string ParserWorker::GetTokenText(const Token& tok) const { return ""; } -Token ParserWorker::NextSignificantToken() { +Token ParserWorker::NextSignificantToken(bool report_error) { if (is_recovery_limit_exceeded()) { return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; } @@ -132,7 +133,7 @@ Token ParserWorker::NextSignificantToken() { if (tok.type == TokenType::kWhitespace || tok.type == TokenType::kComment) { continue; } - if (tok.type == TokenType::kError) { + if (tok.type == TokenType::kError && report_error) { ReportSyntaxError(tok, lexer_.GetError().message); if (is_recovery_limit_exceeded()) { return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; @@ -199,6 +200,36 @@ void ParserWorker::SynchronizeOnDelimiter() { } } +// Checks whether the current identifier is the root of a struct/message +// creation expression (`CreateMessage` in grammar: `'.'? IDENTIFIER ('.' +// IDENTIFIER)* '{' ... '}'`). Unlike standalone identifiers or selector +// chains, reserved identifiers (e.g. `import{}` or `import.Foo{}`) are +// permitted in message type names. +bool ParserWorker::IsStructCreationAhead() { + if (peek_token_.type == TokenType::kLeftBrace) { + return true; + } + if (peek_token_.type != TokenType::kDot) { + return false; + } + const int32_t saved_pos = lexer_.SavePosition(); + auto restore_lexer = absl::MakeCleanup( + [this, saved_pos] { lexer_.RestorePosition(saved_pos); }); + Token tok = peek_token_; + while (tok.type == TokenType::kDot) { + tok = NextSignificantToken(/*report_error=*/false); + if (tok.type != TokenType::kIdent && tok.type != TokenType::kReservedWord) { + return false; + } + // quoted identifiers are not allowed in struct creation expressions + if (tok.start < tok.end && source_.content().at(tok.start) == '`') { + return false; + } + tok = NextSignificantToken(/*report_error=*/false); + } + return tok.type == TokenType::kLeftBrace; +} + int64_t ParserWorker::NextId(int32_t position) { int64_t id = next_id_++; if (id > options_.expression_node_limit && !node_limit_exceeded_) { diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index 80a199989..0aac95710 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -72,11 +72,17 @@ class ParserWorker { const cel::ParserOptions& options() const { return options_; } // Token stream management void InitTokenStream(); - Token NextSignificantToken(); + Token NextSignificantToken(bool report_error = true); Token NextToken(); bool Expect(TokenType type, absl::string_view msg = ""); std::string GetTokenText(const Token& tok) const; void SynchronizeOnDelimiter(); + // Returns true if the tokens following the current identifier form the + // remainder of a struct/message creation type name (e.g. `.Foo.Bar{...}`). + // Used to disambiguate between selectors and struct creation expressions + // based on the grammar rule, which permits CEL reserved words in message type + // names. + bool IsStructCreationAhead(); // ID and Position tracking int64_t NextId(int32_t position); @@ -269,17 +275,24 @@ class PrattParserWorker : public ParserWorker { // frames for each term. void ParseBalancedLogicalChain(ExprNode& lhs, const BinaryOpInfo& op_info); - // Parses prefix unary operators (`!`, `-`) and trailing postfix - // member/indexing operations (`.field`, `[index]`, `.method(args)`). First - // calls `ParseUnary()` to obtain the base operand, then consumes trailing - // selector - // (`.`) and bracket (`[` or `{`) operations in an iterative loop. + // Parses prefix unary operators (`!`, `-`) via `ParseUnaryOps()`, or + // delegates to `ParseMember()` to parse a primary expression and any + // trailing postfix operations. // - // Example (`!a.b[0].c(x)`): `ParseUnary()` consumes `!` and calls - // `ParseSelectorChain()` for `a.b[0].c(x)`, which parses primary `a` and - // then loops iteratively through `.b`, `[0]`, and `.c(x)`. + // Example (`!a.b[0].c(x)`): `ParseUnaryOps()` consumes `!` and calls + // `ParseMember()` for `a.b[0].c(x)`. ExprNode ParseSelectorChain(); + // Parses a primary expression (`ParsePrimary()`) followed by any trailing + // postfix member, indexing, method call, or struct initializer operations + // (`ParseSelectorChainTail()`), capturing the start offset of the member + // expression for source position tracking in optional field selections. + // + // Example (`a.b[0].c(x)`): Calls `ParsePrimary()` to obtain `a`, then calls + // `ParseSelectorChainTail()` to loop iteratively through `.b`, `[0]`, and + // `.c(x)`. + ExprNode ParseMember(); + // Handler for postfix member, index, receiver method, and // struct initializer operations (`.field`, `[index]`, `.method(args)`, // `Type{field: val}`). @@ -289,26 +302,12 @@ class PrattParserWorker : public ParserWorker { // `initial_chain_depth` is the depth already accumulated by `lhs`: a // parenthesized primary such as `(a.b.c)` contributes its own depth, which // the selectors trailing the closing `)` continue to accumulate on top of. - void ParseSelectorChainTail(ExprNode& lhs, int initial_chain_depth); - - // Parses prefix unary operators (logical NOT `!` and negation `-`). If a - // numeric literal immediately follows `-`, folds it directly into a negative - // constant node (`-42`, `-3.14`). Otherwise, wraps the operand in a `_!_` or - // `_-_` function call node. Delegates to `ParseSelectorChain()` for the - // operand to naturally support chained prefix operations (`!!x`, `-!y`), or - // falls back to `ParsePrimary()` if no prefix operator is encountered. - // - // Example (`-42`): Folds directly into a negative integer constant `-42`. - // Example (`!has(x.y)`): Consumes `!` and creates a `LOGICAL_NOT` call node - // wrapping `has(x.y)`. - ExprNode ParseUnary(); + void ParseSelectorChainTail(ExprNode& lhs, int32_t member_start_position, + bool can_be_struct_name, int initial_chain_depth); - // Parses unary operators (`!`, `-`). + // Parses prefix unary operators (`!`, `-`). ExprNode ParseUnaryOps(); - // Parses unary operator chains (`!`, `-`). - ExprNode ParseUnaryOpsChain(Token first_op); - // Parses primary leaf expressions (`nud` atomic atoms), including // parenthesized expressions (`(expr)`), literal constants (`null`, `true`, // `false`, numbers, strings, bytes), collection initializers (`[list]`, @@ -324,7 +323,7 @@ class PrattParserWorker : public ParserWorker { // Example (`(a + b)`): Consumes `(`, recurses to `ParseExpr()`, and expects // `)`. Example (`has(x.y)`): Consumes `has`, parses arguments `(x.y)`, and // expands the `has` macro. - ExprNode ParsePrimary(); + ExprNode ParsePrimary(bool* absl_nullable can_be_struct_name = nullptr); ExprNode ParseList(); ExprNode ParseMap(); @@ -339,8 +338,9 @@ class PrattParserWorker : public ParserWorker { ExprNode ParseBytesLiteral(); void BuildBinaryCall(int64_t op_id, absl::string_view op_name, ExprNode& lhs, ExprNode rhs); - ExprNode ParseIdentOrCall(); - std::string NormalizeIdent(const Token& tok, bool allow_quoted); + ExprNode ParseIdentOrCall(bool* absl_nullable can_be_struct_name = nullptr); + std::string NormalizeIdent(const Token& tok, bool allow_quoted, + bool* absl_nullable is_quoted = nullptr); std::optional ExtractStructName(const ExprNode& expr); int32_t GetLeftmostPosition(const ExprNode& expr); ExprNode BalancedTree(absl::string_view op, std::vector& terms, @@ -491,11 +491,24 @@ void PrattParserWorker::ParseBalancedLogicalChain( template ExprNode PrattParserWorker::ParseSelectorChain() { last_parsed_depth_ = 0; - ExprNode lhs = ParseUnary(); + TokenType tok = peek_token_.type; + if (tok == TokenType::kExclamation || tok == TokenType::kMinus) { + return ParseUnaryOps(); + } + return ParseMember(); +} + +template +ExprNode PrattParserWorker::ParseMember() { + last_parsed_depth_ = 0; + int32_t member_start = peek_token_.start; + bool can_be_struct_name = false; + ExprNode lhs = ParsePrimary(&can_be_struct_name); TokenType tok = peek_token_.type; if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || tok == TokenType::kLeftBrace) { - ParseSelectorChainTail(lhs, last_parsed_depth_); + ParseSelectorChainTail(lhs, member_start, can_be_struct_name, + last_parsed_depth_); } return lhs; } @@ -504,7 +517,8 @@ ExprNode PrattParserWorker::ParseSelectorChain() { // (e.g., `!a.b[0].c(x)`). template void PrattParserWorker::ParseSelectorChainTail( - ExprNode& lhs, int initial_chain_depth) { + ExprNode& lhs, int32_t member_start_position, bool can_be_struct_name, + int initial_chain_depth) { int chain_depth = initial_chain_depth; while (true) { TokenType tok = peek_token_.type; @@ -533,17 +547,24 @@ void PrattParserWorker::ParseSelectorChainTail( return; } bool is_member_call = peek_token_.type == TokenType::kLeftParen; + bool is_quoted = false; std::string id_text = - NormalizeIdent(id_tok, /*allow_quoted=*/!is_member_call); + NormalizeIdent(id_tok, /*allow_quoted=*/!is_member_call, &is_quoted); if (optional) { int64_t op_id = NextId(dot_tok); std::vector args; args.reserve(2); args.push_back(std::move(lhs)); + int64_t field_id = NextId(member_start_position); + SetNodeRange(field_id, member_start_position, + id_tok.end > member_start_position + ? id_tok.end - 1 + : member_start_position); args.push_back( - ast_factory_.NewStringConst(NextId(id_tok), std::move(id_text))); + ast_factory_.NewStringConst(field_id, std::move(id_text))); lhs = ast_factory_.NewCall(op_id, CelOperator::OPT_SELECT, std::move(args)); + can_be_struct_name = false; } else if (peek_token_.type == TokenType::kLeftParen) { Token lparen = NextToken(); int64_t call_id = NextId(lparen); @@ -561,8 +582,10 @@ void PrattParserWorker::ParseSelectorChainTail( // deep. The max preserves the selectors already walked when the // arguments are shallower, as in `a.b.c.f(1)`. chain_depth = std::max(chain_depth, last_parsed_depth_ + 1); + can_be_struct_name = false; } else { lhs = ast_factory_.NewSelect(NextId(dot_tok), std::move(lhs), id_text); + can_be_struct_name = can_be_struct_name && !is_quoted; } } else if (tok == TokenType::kLeftBracket) { if (CheckRecursion(chain_depth)) { @@ -593,7 +616,11 @@ void PrattParserWorker::ParseSelectorChainTail( // max preserves the selectors already walked when the index is shallower, // as in `a.b.c[0]`. chain_depth = std::max(chain_depth, last_parsed_depth_ + 1); + can_be_struct_name = false; } else if (tok == TokenType::kLeftBrace) { + if (!can_be_struct_name) { + break; + } int32_t struct_pos = GetLeftmostPosition(lhs); if (auto struct_name = ExtractStructName(lhs); struct_name.has_value()) { lhs = ParseStruct(NextId(struct_pos), *struct_name); @@ -604,6 +631,7 @@ void PrattParserWorker::ParseSelectorChainTail( // selectors already walked when the fields are shallower, as in // `a.b.Msg{f: 1}`. chain_depth = std::max(chain_depth, last_parsed_depth_); + can_be_struct_name = false; } else { break; } @@ -615,52 +643,47 @@ void PrattParserWorker::ParseSelectorChainTail( } template -ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { +ExprNode PrattParserWorker::ParseUnaryOps() { struct UnaryOp { Token token; int64_t id = 0; }; + Token first_op = NextToken(); + TokenType op_type = first_op.type; std::vector ops; ops.push_back({first_op}); - while (peek_token_.type == TokenType::kExclamation || - peek_token_.type == TokenType::kMinus) { + while (peek_token_.type == op_type) { ops.push_back({NextToken()}); } - const bool has_solitary_trailing_minus = - !ops.empty() && ops.back().token.type == TokenType::kMinus && - (ops.size() == 1 || ops[ops.size() - 2].token.type != TokenType::kMinus); + if (op_type == TokenType::kMinus && ops.size() == 1 && + (peek_token_.type == TokenType::kInt || + peek_token_.type == TokenType::kFloat)) { + ExprNode lhs = (peek_token_.type == TokenType::kInt) + ? ParseNegativeIntLiteral(NextId(first_op)) + : ParseNegativeDoubleLiteral(NextId(first_op)); + TokenType tok = peek_token_.type; + if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || + tok == TokenType::kLeftBrace) { + ParseSelectorChainTail(lhs, first_op.start, + /*can_be_struct_name=*/false, + /*initial_chain_depth=*/0); + } + return lhs; + } if (options_.fold_unary_operators) { - size_t write = 0; - for (size_t read = 0; read < ops.size();) { - size_t next = read; - while (next < ops.size() && - ops[next].token.type == ops[read].token.type) { - next++; - } - if ((next - read) % 2 != 0) { - ops[write++] = ops[read]; - } - read = next; + if (ops.size() % 2 == 0) { + ops.clear(); + } else { + ops.resize(1); } - ops.resize(write); } for (auto& op : ops) { op.id = NextId(op.token); } - const bool is_negative_numeric_literal = - has_solitary_trailing_minus && (peek_token_.type == TokenType::kInt || - peek_token_.type == TokenType::kFloat); - int64_t negative_literal_op_id = 0; - if (is_negative_numeric_literal) { - negative_literal_op_id = ops.back().id; - ops.pop_back(); - } - - ExprNode operand; // Every retained operator wraps the operand in one more call node, so the run // costs as many levels as it has operators even though it is parsed by a // loop. The outermost one is the deepest, so checking it covers the rest. @@ -668,16 +691,27 @@ ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { return ExprNode(); } recursion_depth_ += static_cast(ops.size()); - // Match the ANTLR parser behavior where `-(-)+` prefers to match as - // repeated negate operators instead of a negation of an int literal. - // ---9223372036854775808 will fail to parse. - if (is_negative_numeric_literal) { - operand = (peek_token_.type == TokenType::kInt) - ? ParseNegativeIntLiteral(negative_literal_op_id) - : ParseNegativeDoubleLiteral(negative_literal_op_id); - ParseSelectorChainTail(operand, /*initial_chain_depth=*/0); + + ExprNode operand; + if (op_type == TokenType::kExclamation && + peek_token_.type == TokenType::kMinus) { + Token minus_tok = NextToken(); + if (peek_token_.type == TokenType::kInt) { + operand = ParseNegativeIntLiteral(NextId(minus_tok)); + ParseSelectorChainTail(operand, minus_tok.start, + /*can_be_struct_name=*/false, + /*initial_chain_depth=*/0); + } else if (peek_token_.type == TokenType::kFloat) { + operand = ParseNegativeDoubleLiteral(NextId(minus_tok)); + ParseSelectorChainTail(operand, minus_tok.start, + /*can_be_struct_name=*/false, + /*initial_chain_depth=*/0); + } else { + ReportSyntaxError(minus_tok, "unexpected '-'"); + operand = ParseMember(); + } } else { - operand = ParseSelectorChain(); + operand = ParseMember(); } recursion_depth_ -= static_cast(ops.size()); if (recursion_limit_exceeded_) { @@ -697,57 +731,11 @@ ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { return operand; } -// Parses prefix unary operators (`!`, `-`) iteratively and folds negative -// numeric literals (e.g., `-42`, `!has(x.y)`). -template -ExprNode PrattParserWorker::ParseUnary() { - TokenType tok = peek_token_.type; - if (tok == TokenType::kExclamation || tok == TokenType::kMinus) { - return ParseUnaryOps(); - } - return ParsePrimary(); -} - -template -ExprNode PrattParserWorker::ParseUnaryOps() { - Token op = NextToken(); - TokenType op_type = op.type; - if (peek_token_.type == TokenType::kExclamation || - peek_token_.type == TokenType::kMinus) { - return ParseUnaryOpsChain(op); - } - - if (op_type == TokenType::kMinus) { - if (peek_token_.type == TokenType::kInt) { - return ParseNegativeIntLiteral(NextId(op)); - } - if (peek_token_.type == TokenType::kFloat) { - return ParseNegativeDoubleLiteral(NextId(op)); - } - } - - if (CheckRecursion(0)) { - return ExprNode(); - } - int64_t op_id = NextId(op); - recursion_depth_++; - ExprNode operand = ParseSelectorChain(); - recursion_depth_--; - if (recursion_limit_exceeded_) { - return ExprNode(); - } - std::vector args; - args.push_back(std::move(operand)); - absl::string_view op_name = (op_type == TokenType::kExclamation) - ? CelOperator::LOGICAL_NOT - : CelOperator::NEGATE; - return ast_factory_.NewCall(op_id, std::string(op_name), std::move(args)); -} - // Parses identifiers (e.g., `foo`, `.foo`) and global function or macro calls // (e.g., `foo(args)`, `has(x.y)`). template -ExprNode PrattParserWorker::ParseIdentOrCall() { +ExprNode PrattParserWorker::ParseIdentOrCall( + bool* absl_nullable can_be_struct_name) { TokenType tok_type = peek_token_.type; bool leading_dot = false; Token first_tok = peek_token_; @@ -763,9 +751,11 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { } return ast_factory_.NewUnspecified(NextId(id_tok)); } - std::string id_text = NormalizeIdent(id_tok, /*allow_quoted=*/false); + bool is_quoted = false; + std::string id_text = + NormalizeIdent(id_tok, /*allow_quoted=*/false, &is_quoted); if (ABSL_PREDICT_FALSE(id_tok.type == TokenType::kReservedWord)) { - if (cel::internal::LexisIsReserved(id_text)) { + if (cel::internal::LexisIsReserved(id_text) && !IsStructCreationAhead()) { ReportError(id_tok, absl::StrFormat("reserved identifier: %s", id_text)); } } @@ -781,6 +771,9 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { } return ast_factory_.NewCall(call_id, name, std::move(args)); } + if (can_be_struct_name != nullptr) { + *can_be_struct_name = !is_quoted; + } int64_t id = NextId(leading_dot ? first_tok : id_tok); return ast_factory_.NewIdent(id, std::move(name)); } @@ -791,7 +784,11 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { // (`[...]`, `{...}`), and identifiers/global function calls (`foo`, // `has(x.y)`). template -ExprNode PrattParserWorker::ParsePrimary() { +ExprNode PrattParserWorker::ParsePrimary( + bool* absl_nullable can_be_struct_name) { + if (can_be_struct_name != nullptr) { + *can_be_struct_name = false; + } switch (peek_token_.type) { case TokenType::kLeftParen: { if (recursion_limit_exceeded_ || is_recovery_limit_exceeded()) { @@ -806,8 +803,11 @@ ExprNode PrattParserWorker::ParsePrimary() { // selectors or binary/ternary operators belonging to that enclosing // parenthesized level using the already-parsed inner expression as the // LHS. - int open_parens = 0; + int open_parens = 1; + NextToken(); + std::vector extra_paren_starts; while (peek_token_.type == TokenType::kLeftParen) { + extra_paren_starts.push_back(peek_token_.start); open_parens++; NextToken(); } @@ -828,9 +828,10 @@ ExprNode PrattParserWorker::ParsePrimary() { Expect(TokenType::kRightParen); if (i < open_parens - 1 && peek_token_.type != TokenType::kRightParen) { TokenType tok = peek_token_.type; - if (tok == TokenType::kDot || tok == TokenType::kLeftBracket || - tok == TokenType::kLeftBrace) { - ParseSelectorChainTail(expr, chain_depth); + if (tok == TokenType::kDot || tok == TokenType::kLeftBracket) { + ParseSelectorChainTail(expr, + extra_paren_starts[open_parens - 2 - i], + /*can_be_struct_name=*/false, chain_depth); } ParseBinaryAndTernaryFromLhs(expr, 0, last_parsed_depth_); chain_depth = last_parsed_depth_; @@ -865,7 +866,7 @@ ExprNode PrattParserWorker::ParsePrimary() { case TokenType::kDot: case TokenType::kIdent: case TokenType::kReservedWord: - return ParseIdentOrCall(); + return ParseIdentOrCall(can_be_struct_name); default: { Token bad_tok = NextToken(); if (bad_tok.type != TokenType::kError) { @@ -1147,11 +1148,15 @@ ExprNode PrattParserWorker::ParseBytesLiteral() { // Normalizes regular & quoted identifiers (e.g., `foo`, `` `quoted.ident` ``). template -std::string PrattParserWorker::NormalizeIdent(const Token& tok, - bool allow_quoted) { +std::string PrattParserWorker::NormalizeIdent( + const Token& tok, bool allow_quoted, bool* absl_nullable is_quoted) { std::string text = GetTokenText(tok); - if (text.empty()) return ""; + if (text.empty()) { + if (is_quoted != nullptr) *is_quoted = false; + return ""; + } if (text.front() == '`') { + if (is_quoted != nullptr) *is_quoted = true; if (!allow_quoted) { ReportError(tok, "unexpected quoted identifier"); return ""; @@ -1179,6 +1184,7 @@ std::string PrattParserWorker::NormalizeIdent(const Token& tok, } return std::string(inner); } + if (is_quoted != nullptr) *is_quoted = false; return std::string(text); } diff --git a/parser/parser_test.cc b/parser/parser_test.cc index 3512e1819..90838cc65 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -1508,6 +1508,10 @@ std::vector test_cases = { "Msg{\n " "?field:value^#3:Expr.Ident#^#2:Expr.CreateStruct.Entry#\n}^#1:Expr." "CreateStruct#"}, + {"import{}", "import{}^#1:Expr.CreateStruct#"}, + {".import{}", ".import{}^#1:Expr.CreateStruct#"}, + {"import.Foo{}", "import.Foo{}^#1:Expr.CreateStruct#"}, + {"Foo.import{}", "Foo.import{}^#1:Expr.CreateStruct#"}, {"m.optMap(v, f)", "_?_:_(\n m^#1:Expr.Ident#.hasValue()^#6:Expr.Call#,\n optional.of(\n " " __comprehension__(\n // Variable\n #unused,\n // " @@ -1649,7 +1653,149 @@ std::vector test_cases = { " | ^\n" "ERROR: :1:8: Syntax error: unexpected token after expression\n" " | \"hello\rworld\"\n" - " | .......^"}}; + " | .......^"}, + {"a.?b.?c", + "_?._(\n" + " _?._(\n" + " a^#1:Expr.Ident#,\n" + " \"b\"^#3:string#\n" + " )^#2:Expr.Call#,\n" + " \"c\"^#5:string#\n" + ")^#4:Expr.Call#", + "", + "_?._(\n" + " _?._(\n" + " a^#1[1,0]#,\n" + " \"b\"^#3[1,0]#\n" + " )^#2[1,1]#,\n" + " \"c\"^#5[1,0]#\n" + ")^#4[1,4]#"}, + {"!-42", + "!_(\n" + " -42^#2:int64#\n" + ")^#1:Expr.Call#", + "", + "!_(\n" + " -42^#2[1,1]#\n" + ")^#1[1,0]#"}, + {"!-4.2", + "!_(\n" + " -4.2^#2:double#\n" + ")^#1:Expr.Call#", + "", + "!_(\n" + " -4.2^#2[1,1]#\n" + ")^#1[1,0]#"}, + {"9in-x", + "@in(\n" + " 9^#1:int64#,\n" + " -_(\n" + " x^#4:Expr.Ident#\n" + " )^#3:Expr.Call#\n" + ")^#2:Expr.Call#"}, + {"1 \v + 2", "", + "ERROR: :1:3: Syntax error: token recognition error at: '\v'\n" + " | 1 \v + 2\n" + " | ..^", + "", "", "", + "ERROR: :1:3: Syntax error: unexpected character\n" + " | 1 \v + 2\n" + " | ..^"}, + {"-!x", "", + "ERROR: :1:2: Syntax error: no viable alternative at input '-!'\n" + " | -!x\n" + " | .^", + "", "", "", + "ERROR: :1:2: Syntax error: unexpected token\n" + " | -!x\n" + " | .^\n" + "ERROR: :1:3: Syntax error: unexpected token after expression\n" + " | -!x\n" + " | ..^"}, + {"!-x", "", + "ERROR: :1:2: invalid constant literal expression\n" + " | !-x\n" + " | .^\n" + "ERROR: :1:3: Syntax error: no viable alternative at input '-x'\n" + " | !-x\n" + " | ..^", + "", "", "", + "ERROR: :1:2: Syntax error: unexpected '-'\n" + " | !-x\n" + " | .^"}, + {"a.in", "", + "ERROR: :1:3: Syntax error: no viable alternative at input '.in'\n" + " | a.in\n" + " | ..^\n" + "ERROR: :1:5: Syntax error: mismatched input '' expecting " + "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " + "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + " | a.in\n" + " | ....^", + "", "", "", + "ERROR: :1:3: Syntax error: expected identifier after '.'\n" + " | a.in\n" + " | ..^"}, + {"has(a.`$b`)", "", + "ERROR: :1:7: Syntax error: token recognition error at: '`$'\n" + " | has(a.`$b`)\n" + " | ......^\n" + "ERROR: :1:10: Syntax error: token recognition error at: '`)'\n" + " | has(a.`$b`)\n" + " | .........^\n" + "ERROR: :1:12: Syntax error: missing ')' at ''\n" + " | has(a.`$b`)\n" + " | ...........^", + "", "", "", + "ERROR: :1:7: unexpected quoted identifier\n" + " | has(a.`$b`)\n" + " | ......^"}, + {"(a){}", "", + "ERROR: :1:4: Syntax error: mismatched input '{' expecting \n" + " | (a){}\n" + " | ...^", + "", "", "", + "ERROR: :1:4: Syntax error: unexpected token after expression\n" + " | (a){}\n" + " | ...^"}, + {"(a.b){}", "", + "ERROR: :1:6: Syntax error: mismatched input '{' expecting \n" + " | (a.b){}\n" + " | .....^", + "", "", "", + "ERROR: :1:6: Syntax error: unexpected token after expression\n" + " | (a.b){}\n" + " | .....^"}, + {"(a).b{}", "", + "ERROR: :1:6: Syntax error: mismatched input '{' expecting \n" + " | (a).b{}\n" + " | .....^", + "", "", "", + "ERROR: :1:6: Syntax error: unexpected token after expression\n" + " | (a).b{}\n" + " | .....^"}, + {"a.`b-c`{}", "", + "ERROR: :1:8: Syntax error: mismatched input '{' expecting \n" + " | a.`b-c`{}\n" + " | .......^", + "", "", "", + "ERROR: :1:8: Syntax error: unexpected token after expression\n" + " | a.`b-c`{}\n" + " | .......^"}, + {"Msg{`$b`: 1}", "", + "ERROR: :1:5: Syntax error: token recognition error at: '`$'\n" + " | Msg{`$b`: 1}\n" + " | ....^\n" + "ERROR: :1:8: Syntax error: token recognition error at: '`:'\n" + " | Msg{`$b`: 1}\n" + " | .......^\n" + "ERROR: :1:11: Syntax error: missing ':' at '1'\n" + " | Msg{`$b`: 1}\n" + " | ..........^", + "", "", "", + "ERROR: :1:5: unexpected quoted identifier\n" + " | Msg{`$b`: 1}\n" + " | ....^"}}; absl::string_view ConstantKind(const cel::Constant& c) { switch (c.kind_case()) { @@ -1866,7 +2012,7 @@ TEST_P(ExpressionTest, Parse) { << result->parsed_expr().ShortDebugString(); } - if (!options_.enable_pratt_parser && !test_info.L.empty()) { + if (!test_info.L.empty()) { LocationAdorner location_adorner(result->parsed_expr().source_info()); ExprPrinter w(location_adorner); std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); From 931a8e6effc67906ef0bbab3b3b85582950746b3 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 24 Sep 2026 11:22:32 -0700 Subject: [PATCH 236/350] Limit Trace call to Expr IDs in range [0, std::numeric_limits::max()]. PiperOrigin-RevId: 987635321 --- eval/eval/evaluator_core.cc | 10 +++++++++- eval/public/cel_expression.h | 7 +++++++ runtime/runtime.h | 7 +++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 05dbed854..a7f8e74ea 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -15,6 +15,8 @@ #include "eval/eval/evaluator_core.h" #include +#include +#include #include #include @@ -132,7 +134,13 @@ absl::StatusOr ExecutionFrame::Evaluate( "Try to disable short-circuiting."; continue; } - if (EvaluationStatus status(listener(expr->id(), value_stack().Peek(), + const int64_t id = expr->id(); + // Skip if the id is out of range. + // Will take advantage of this in a follow up to bit pack the id. + if (id < 0 || id > std::numeric_limits::max()) { + continue; + } + if (EvaluationStatus status(listener(id, value_stack().Peek(), descriptor_pool(), message_factory(), arena())); !status.ok()) { diff --git a/eval/public/cel_expression.h b/eval/public/cel_expression.h index 4cf029e89..af28e2ae6 100644 --- a/eval/public/cel_expression.h +++ b/eval/public/cel_expression.h @@ -24,6 +24,13 @@ namespace google::api::expr::runtime { // then the order of the callback invocations is guaranteed to correspond // the order of variable sub-elements (e.g. the order of elements returned // by Comprehension.iter_range). +// +// Expression IDs outside of the range [0, INT32_MAX] are not supported and +// will not invoke the listener. While the AST allows any int64, supported +// parser implementations should use a dense range starting at 1. In practice, +// no AST should contain more than ~ 1e9 nodes. +// +// ID 0 should not be considered valid, but is supported for legacy reasons. using CelEvaluationListener = std::function; diff --git a/runtime/runtime.h b/runtime/runtime.h index 8c76236dd..02ad9fd06 100644 --- a/runtime/runtime.h +++ b/runtime/runtime.h @@ -130,6 +130,13 @@ class TraceableProgram : public Program { // to an AST expression node. The value provided is the top of the value // stack, corresponding to the result of evaluating the given sub expression. // + // Expression IDs outside of the range [0, INT32_MAX] are not supported and + // will not invoke the listener. While the AST allows any int64, supported + // parser implementations should use a dense range starting at 1. In practice, + // no AST should contain more than ~ 1e9 nodes. + // + // ID 0 should not be considered valid, but is supported for legacy reasons. + // // A returning a non-ok status stops evaluation and forwards the error. using EvaluationListener = absl::AnyInvocable Date: Thu, 24 Sep 2026 13:53:57 -0700 Subject: [PATCH 237/350] Add utility to clone `CelValue` onto the same or different `google::protobuf::Arena` PiperOrigin-RevId: 987735736 --- eval/public/BUILD | 22 +++++ eval/public/cel_value_clone.cc | 166 +++++++++++++++++++++++++++++++++ eval/public/cel_value_clone.h | 40 ++++++++ 3 files changed, 228 insertions(+) create mode 100644 eval/public/cel_value_clone.cc create mode 100644 eval/public/cel_value_clone.h diff --git a/eval/public/BUILD b/eval/public/BUILD index 0c9c6f23e..a0a3bcb27 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -114,6 +114,28 @@ cc_library( ], ) +cc_library( + name = "cel_value_clone", + srcs = ["cel_value_clone.cc"], + hdrs = ["cel_value_clone.h"], + deps = [ + ":cel_value", + ":message_wrapper", + ":unknown_set", + "//eval/public/containers:container_backed_list_impl", + "//eval/public/containers:container_backed_map_impl", + "//internal:status_macros", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", + "@com_google_absl//absl/time", + "@com_google_absl//absl/types:optional", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "cel_attribute", srcs = [ diff --git a/eval/public/cel_value_clone.cc b/eval/public/cel_value_clone.cc new file mode 100644 index 000000000..8a07033df --- /dev/null +++ b/eval/public/cel_value_clone.cc @@ -0,0 +1,166 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eval/public/cel_value_clone.h" + +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" +#include "absl/time/time.h" +#include "absl/types/optional.h" +#include "eval/public/cel_value.h" +#include "eval/public/containers/container_backed_list_impl.h" +#include "eval/public/containers/container_backed_map_impl.h" +#include "eval/public/message_wrapper.h" +#include "eval/public/unknown_set.h" +#include "internal/status_macros.h" +#include "google/protobuf/arena.h" + +namespace google::api::expr::runtime { + +namespace { + +struct CloneVisitor { + google::protobuf::Arena* const arena; + + absl::StatusOr operator()(bool arg) { + return CelValue::CreateBool(arg); + } + absl::StatusOr operator()(int64_t arg) { + return CelValue::CreateInt64(arg); + } + absl::StatusOr operator()(uint64_t arg) { + return CelValue::CreateUint64(arg); + } + absl::StatusOr operator()(double arg) { + return CelValue::CreateDouble(arg); + } + absl::StatusOr operator()(CelValue::NullType) { + return CelValue::CreateNull(); + } + + absl::StatusOr operator()(CelValue::StringHolder arg) { + if (arg.value().empty()) { + return CelValue::CreateStringView(""); + } + return CelValue::CreateString(CelValue::StringHolder( + google::protobuf::Arena::Create(arena, arg.value()))); + } + + absl::StatusOr operator()(CelValue::BytesHolder arg) { + if (arg.value().empty()) { + return CelValue::CreateBytesView(""); + } + return CelValue::CreateBytes(CelValue::BytesHolder( + google::protobuf::Arena::Create(arena, arg.value()))); + } + + absl::StatusOr operator()(const MessageWrapper& arg) { + if (arg.HasFullProto()) { + google::protobuf::Message* dup = + static_cast(arg.message_ptr())->New(arena); + dup->CopyFrom(*static_cast(arg.message_ptr())); + return CelValue::CreateMessageWrapper( + MessageWrapper(dup, arg.legacy_type_info())); + } + google::protobuf::MessageLite* dup = arg.message_ptr()->New(arena); + absl::Cord serialized; + if (!arg.message_ptr()->SerializePartialToCord(&serialized)) { + return absl::UnknownError( + absl::StrCat("failed to serialize message: ", dup->GetTypeName())); + } + if (!dup->ParsePartialFromString(serialized)) { + return absl::UnknownError( + absl::StrCat("failed to parse message: ", dup->GetTypeName())); + } + return CelValue::CreateMessageWrapper( + MessageWrapper(dup, arg.legacy_type_info())); + } + + absl::StatusOr operator()(absl::Duration arg) { + return CelValue::CreateUncheckedDuration(arg); + } + + absl::StatusOr operator()(absl::Time arg) { + return CelValue::CreateTimestamp(arg); + } + + absl::StatusOr operator()(const CelList* arg) { + if (arg->empty()) { + return CelValue::CreateList(); + } + std::vector elements; + elements.reserve(arg->size()); + for (int i = 0; i < arg->size(); i++) { + CEL_ASSIGN_OR_RETURN(auto element, + CelValueClone(arena, arg->Get(arena, i))); + elements.push_back(element); + } + return CelValue::CreateList(google::protobuf::Arena::Create( + arena, std::move(elements))); + } + + absl::StatusOr operator()(const CelMap* arg) { + if (arg->empty()) { + return CelValue::CreateMap(); + } + CEL_ASSIGN_OR_RETURN(auto keys, arg->ListKeys(arena)); + CelMapBuilder* builder = google::protobuf::Arena::Create(arena); + for (int i = 0; i < keys->size(); ++i) { + CelValue key = keys->Get(arena, i); + absl::optional value = arg->Get(arena, key); + if (!value.has_value()) { + return absl::UnknownError( + "CelMap::ListKeys returned key but CelMap::Get did not find the " + "entry"); + } + CEL_ASSIGN_OR_RETURN(auto key_dup, CelValueClone(arena, key)); + CEL_ASSIGN_OR_RETURN(auto value_dup, CelValueClone(arena, *value)); + CEL_RETURN_IF_ERROR(builder->Add(key_dup, value_dup)); + } + return CelValue::CreateMap(builder); + } + + absl::StatusOr operator()(const UnknownSet* arg) { + return CelValue::CreateUnknownSet( + google::protobuf::Arena::Create(arena, *arg)); + } + + absl::StatusOr operator()(CelValue::CelTypeHolder arg) { + return CelValue::CreateCelType(CelValue::CelTypeHolder( + google::protobuf::Arena::Create(arena, arg.value()))); + } + + absl::StatusOr operator()(const CelError* arg) { + return CelValue::CreateError(google::protobuf::Arena::Create(arena, *arg)); + } +}; + +} // namespace + +absl::StatusOr CelValueClone(google::protobuf::Arena* arena, CelValue in) { + ABSL_DCHECK(arena != nullptr); + return in.Visit>(CloneVisitor{.arena = arena}); +} + +} // namespace google::api::expr::runtime diff --git a/eval/public/cel_value_clone.h b/eval/public/cel_value_clone.h new file mode 100644 index 000000000..2ffe47894 --- /dev/null +++ b/eval/public/cel_value_clone.h @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_CEL_VALUE_CLONE_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_CEL_VALUE_CLONE_H_ + +#include "absl/status/statusor.h" +#include "eval/public/cel_value.h" +#include "google/protobuf/arena.h" + +namespace google::api::expr::runtime { + +// Clones (deep copy) a CelValue such that the resulting CelValue has a lifetime +// tied to the given arena. The container implementations for the output are not +// guaranteed to be the same as the input. +absl::StatusOr CelValueClone(google::protobuf::Arena* arena, CelValue in); +inline absl::StatusOr CelValueClone(google::protobuf::Arena* arena, + absl::StatusOr in) { + if (!in.ok()) { + return in; + } + return CelValueClone(arena, *in); +} + +} // namespace google::api::expr::runtime + +#endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_CEL_VALUE_CLONE_H_ From cd4143cb80c921fdf144b080031304b5d29fee6d Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Fri, 25 Sep 2026 09:37:11 -0700 Subject: [PATCH 238/350] No public description PiperOrigin-RevId: 988329985 --- checker/internal/type_checker_impl.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/checker/internal/type_checker_impl.cc b/checker/internal/type_checker_impl.cc index 0e71e864a..5e9272d10 100644 --- a/checker/internal/type_checker_impl.cc +++ b/checker/internal/type_checker_impl.cc @@ -610,9 +610,11 @@ void ResolveVisitor::PostVisitMap(const Expr& expr, const MapExpr& map) { for (const auto& entry : map.entries()) { const Expr* value = &entry.value(); Type value_type = GetDeducedType(value); + OptionalType opt_value_type; if (entry.optional()) { if (value_type.IsOptional()) { - value_type = value_type.GetOptional().GetParameter(); + opt_value_type = value_type.GetOptional(); + value_type = opt_value_type.GetParameter(); } else { ReportTypeMismatch(entry.value().id(), OptionalType(arena_, value_type), value_type); @@ -648,9 +650,11 @@ void ResolveVisitor::PostVisitList(const Expr& expr, const ListExpr& list) { for (const auto& element : list.elements()) { const Expr* value = &element.expr(); Type value_type = GetDeducedType(value); + OptionalType opt_value_type; if (element.optional()) { if (value_type.IsOptional()) { - value_type = value_type.GetOptional().GetParameter(); + opt_value_type = value_type.GetOptional(); + value_type = opt_value_type.GetParameter(); } else { ReportTypeMismatch(element.expr().id(), OptionalType(arena_, value_type), value_type); @@ -1211,8 +1215,10 @@ void ResolveVisitor::HandleOptSelect(const Expr& expr) { } Type operand_type = GetDeducedType(operand); + OptionalType opt_operand_type; if (operand_type.IsOptional()) { - operand_type = operand_type.GetOptional().GetParameter(); + opt_operand_type = operand_type.GetOptional(); + operand_type = opt_operand_type.GetParameter(); } std::optional field_type = CheckFieldType( From addea6d6384b825984b39c715c92672ac510c293 Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 25 Sep 2026 14:38:09 -0700 Subject: [PATCH 239/350] Cache evaluation option flags in ExecutionFrameBase PiperOrigin-RevId: 988502087 --- eval/eval/evaluator_core.h | 42 +++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 575abfa05..d1d4e5438 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -178,7 +178,18 @@ class ExecutionFrameBase { activation.GetMissingAttributes()), slots_(&ComprehensionSlots::GetEmptyInstance()), max_iterations_(options.comprehension_max_iterations), - iterations_(0) { + iterations_(0), + attribute_tracking_enabled_( + options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled || + options_->enable_missing_attribute_errors), + missing_attribute_errors_enabled_( + options_->enable_missing_attribute_errors), + unknown_processing_enabled_(options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled), + unknown_function_results_enabled_( + options_->unknown_processing == + cel::UnknownProcessingOptions::kAttributeAndFunction) { if (unknown_processing_enabled()) { if (auto matcher = cel::runtime_internal:: ActivationAttributeMatcherAccess::GetAttributeMatcher(activation); @@ -209,7 +220,18 @@ class ExecutionFrameBase { activation.GetMissingAttributes()), slots_(&slots), max_iterations_(options.comprehension_max_iterations), - iterations_(0) { + iterations_(0), + attribute_tracking_enabled_( + options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled || + options_->enable_missing_attribute_errors), + missing_attribute_errors_enabled_( + options_->enable_missing_attribute_errors), + unknown_processing_enabled_(options_->unknown_processing != + cel::UnknownProcessingOptions::kDisabled), + unknown_function_results_enabled_( + options_->unknown_processing == + cel::UnknownProcessingOptions::kAttributeAndFunction) { if (unknown_processing_enabled()) { if (auto matcher = cel::runtime_internal:: ActivationAttributeMatcherAccess::GetAttributeMatcher(activation); @@ -246,23 +268,19 @@ class ExecutionFrameBase { } bool attribute_tracking_enabled() const { - return options_->unknown_processing != - cel::UnknownProcessingOptions::kDisabled || - options_->enable_missing_attribute_errors; + return attribute_tracking_enabled_; } bool missing_attribute_errors_enabled() const { - return options_->enable_missing_attribute_errors; + return missing_attribute_errors_enabled_; } bool unknown_processing_enabled() const { - return options_->unknown_processing != - cel::UnknownProcessingOptions::kDisabled; + return unknown_processing_enabled_; } bool unknown_function_results_enabled() const { - return options_->unknown_processing == - cel::UnknownProcessingOptions::kAttributeAndFunction; + return unknown_function_results_enabled_; } ComprehensionSlots& comprehension_slots() { return *slots_; } @@ -294,6 +312,10 @@ class ExecutionFrameBase { ComprehensionSlots* absl_nonnull slots_; const int max_iterations_; int iterations_; + const bool attribute_tracking_enabled_; + const bool missing_attribute_errors_enabled_; + const bool unknown_processing_enabled_; + const bool unknown_function_results_enabled_; }; // ExecutionFrame manages the context needed for expression evaluation. From 46ca9cac8eceb170df22606e97c6894cabe0de80 Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 28 Sep 2026 12:24:08 -0700 Subject: [PATCH 240/350] Migrate some from `cel::ErrorValue()` to `cel::ErrorValue::From()` PiperOrigin-RevId: 989772926 --- codelab/network_functions.cc | 17 +- common/BUILD | 1 + common/legacy_value.cc | 13 +- common/value.cc | 80 +++-- common/value.h | 3 + common/values/custom_map_value.cc | 18 +- common/values/error_value.h | 6 + common/values/optional_value.cc | 8 +- common/values/parsed_json_value.cc | 5 +- common/values/parsed_message_value.cc | 2 +- eval/eval/BUILD | 2 +- eval/eval/comprehension_step.cc | 32 +- eval/eval/container_access_step.cc | 42 ++- eval/eval/create_struct_step.cc | 10 +- eval/eval/equality_steps.cc | 16 +- eval/eval/function_step.cc | 21 +- eval/eval/ident_step.cc | 6 +- eval/eval/jump_step.cc | 4 +- eval/eval/logic_step.cc | 29 +- eval/eval/optional_or_step.cc | 22 +- eval/eval/select_step.cc | 20 +- eval/eval/ternary_step.cc | 9 +- extensions/BUILD | 3 +- extensions/encoders.cc | 3 +- extensions/formatting.cc | 11 +- extensions/lists_functions.cc | 62 ++-- extensions/math_ext.cc | 56 +-- extensions/regex_ext.cc | 33 +- extensions/regex_functions.cc | 21 +- extensions/select_optimization.cc | 25 +- extensions/strings.cc | 15 +- runtime/BUILD | 1 + runtime/optional_types.cc | 24 +- runtime/standard/BUILD | 4 + runtime/standard/arithmetic_functions.cc | 91 ++--- runtime/standard/equality_functions.cc | 21 +- runtime/standard/logical_functions.cc | 8 +- runtime/standard/time_functions.cc | 325 +++++++++++------- runtime/standard/type_conversion_functions.cc | 104 +++--- 39 files changed, 720 insertions(+), 453 deletions(-) diff --git a/codelab/network_functions.cc b/codelab/network_functions.cc index 6cc1505a9..170e4243b 100644 --- a/codelab/network_functions.cc +++ b/codelab/network_functions.cc @@ -312,7 +312,8 @@ cel::Value parseAddress( absl::string_view addr = str.ToStringView(&buf); std::optional rep = NetworkAddressRep::Parse(addr); if (!rep.has_value()) { - return cel::ErrorValue(absl::InvalidArgumentError("invalid address")); + return cel::ErrorValue::From(absl::InvalidArgumentError("invalid address"), + arena); } return NetworkAddressRep::MakeValue(*rep); } @@ -337,21 +338,25 @@ cel::Value parseAddressMatcher( absl::string_view addr = str.ToStringView(&buf); std::optional rep = NetworkAddressMatcher::Parse(addr); if (!rep.has_value()) { - return cel::ErrorValue( - absl::InvalidArgumentError("invalid address matcher")); + return cel::ErrorValue::From( + absl::InvalidArgumentError("invalid address matcher"), arena); } return NetworkAddressMatcher::MakeValue(arena, std::move(rep).value()); } -cel::Value containsAddress(const cel::OpaqueValue& matcher, - const cel::OpaqueValue& addr) { +cel::Value containsAddress( + const cel::OpaqueValue& matcher, const cel::OpaqueValue& addr, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + google::protobuf::Arena* absl_nonnull arena) { const auto* matcher_rep = NetworkAddressMatcher::Unwrap(matcher); auto addr_rep = NetworkAddressRep::Unwrap(addr); if (matcher_rep == nullptr || !addr_rep.has_value()) { // dispatcher should catch this, but right now only distiguishes at the // kind level. - return cel::ErrorValue(absl::InvalidArgumentError("no matching overload")); + return cel::ErrorValue::From( + absl::InvalidArgumentError("no matching overload"), arena); } return cel::BoolValue(matcher_rep->Match(*addr_rep)); } diff --git a/common/BUILD b/common/BUILD index f34f07fcf..ba9f2db54 100644 --- a/common/BUILD +++ b/common/BUILD @@ -831,6 +831,7 @@ cc_library( "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/time", "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:source_location", "@com_google_absl//absl/types:span", "@com_google_absl//absl/types:variant", "@com_google_absl//absl/utility", diff --git a/common/legacy_value.cc b/common/legacy_value.cc index c2dc1cc7f..59f9db278 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -532,7 +532,8 @@ absl::Status LegacyListValue::Get( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { if (ABSL_PREDICT_FALSE(index < 0 || index >= impl_->size())) { - *result = ErrorValue(absl::InvalidArgumentError("index out of bounds")); + *result = ErrorValue::From( + absl::InvalidArgumentError("index out of bounds"), arena); return absl::OkStatus(); } CEL_RETURN_IF_ERROR( @@ -714,7 +715,7 @@ absl::Status LegacyMapValue::Get( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); return absl::OkStatus(); } CEL_ASSIGN_OR_RETURN(auto cel_key, LegacyValue(arena, key)); @@ -747,7 +748,7 @@ absl::StatusOr LegacyMapValue::Find( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); } CEL_ASSIGN_OR_RETURN(auto cel_key, LegacyValue(arena, key)); auto cel_value = impl_->Get(arena, cel_key); @@ -779,13 +780,13 @@ absl::Status LegacyMapValue::Has( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); return absl::OkStatus(); } CEL_ASSIGN_OR_RETURN(auto cel_key, LegacyValue(arena, key)); absl::StatusOr has = impl_->Has(cel_key); if (!has.ok()) { - *result = ErrorValue(std::move(has).status()); + *result = ErrorValue::From(std::move(has).status(), arena); return absl::OkStatus(); } @@ -1192,7 +1193,7 @@ absl::StatusOr FromLegacyValue(google::protobuf::Arena* arena, return CreateTypeValueFromView(arena, legacy_value.CelTypeOrDie().value()); case CelValue::Type::kError: - return ErrorValue(*legacy_value.ErrorOrDie()); + return ErrorValue::From(*legacy_value.ErrorOrDie(), arena); case CelValue::Type::kAny: return absl::InternalError(absl::StrCat( "illegal attempt to convert special CelValue type ", diff --git a/common/value.cc b/common/value.cc index a2ffea620..3feadeb17 100644 --- a/common/value.cc +++ b/common/value.cc @@ -26,6 +26,7 @@ #include "google/protobuf/struct.pb.h" #include "absl/base/attributes.h" +#include "absl/base/no_destructor.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/functional/overload.h" @@ -39,6 +40,7 @@ #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "absl/types/optional.h" +#include "absl/types/source_location.h" #include "absl/types/variant.h" #include "common/allocator.h" #include "common/memory.h" @@ -322,12 +324,20 @@ Value NonNullEnumValue(const google::protobuf::EnumValueDescriptor* absl_nonnull } Value NonNullEnumValue(const google::protobuf::EnumDescriptor* absl_nonnull type, - int32_t number) { + int32_t number, google::protobuf::Arena* absl_nullable arena) { ABSL_DCHECK(type != nullptr); if (type->is_closed()) { if (ABSL_PREDICT_FALSE(type->FindValueByNumber(number) == nullptr)) { - return ErrorValue(absl::InvalidArgumentError(absl::StrCat( - "closed enum has no such value: ", type->full_name(), ".", number))); + if (arena == nullptr) { + static const absl::NoDestructor error( + absl::InvalidArgumentError("closed enum has no such value", + absl::SourceLocation())); + return ErrorValue::WrapUnsafe(&*error); + } + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "closed enum has no such value: ", + type->full_name(), ".", number)), + arena); } } return IntValue(number); @@ -351,7 +361,18 @@ Value Value::Enum(const google::protobuf::EnumDescriptor* absl_nonnull type, ABSL_DCHECK_EQ(number, 0); return NullValue(); } - return NonNullEnumValue(type, number); + return NonNullEnumValue(type, number, nullptr); +} + +Value Value::Enum(const google::protobuf::EnumDescriptor* absl_nonnull type, + int32_t number, google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(type != nullptr); + ABSL_DCHECK(arena != nullptr); + if (type->full_name() == "google.protobuf.NullValue") { + ABSL_DCHECK_EQ(number, 0); + return NullValue(); + } + return NonNullEnumValue(type, number, arena); } namespace common_internal { @@ -669,7 +690,7 @@ void EnumMapFieldValueAccessor( ABSL_DCHECK(!field->is_repeated()); ABSL_DCHECK_EQ(field->cpp_type(), google::protobuf::FieldDescriptor::CPPTYPE_ENUM); - *result = NonNullEnumValue(field->enum_type(), value.GetEnumValue()); + *result = NonNullEnumValue(field->enum_type(), value.GetEnumValue(), arena); } void NullMapFieldValueAccessor( @@ -1044,7 +1065,7 @@ void EnumRepeatedFieldAccessor( *result = NonNullEnumValue( field->enum_type(), - reflection->GetRepeatedEnumValue(*message, field, index)); + reflection->GetRepeatedEnumValue(*message, field, index), arena); } void NullRepeatedFieldAccessor( @@ -1363,7 +1384,7 @@ Value Value::FromMessage( auto status_or_adapted = well_known_types::AdaptFromMessage( arena, message, descriptor_pool, message_factory, scratch); if (ABSL_PREDICT_FALSE(!status_or_adapted.ok())) { - return ErrorValue(std::move(status_or_adapted).status()); + return ErrorValue::From(std::move(status_or_adapted).status(), arena); } return absl::visit( absl::Overload(OwningWellKnownTypesValueVisitor{ @@ -1391,7 +1412,7 @@ Value Value::FromMessage( auto status_or_adapted = well_known_types::AdaptFromMessage( arena, message, descriptor_pool, message_factory, scratch); if (ABSL_PREDICT_FALSE(!status_or_adapted.ok())) { - return ErrorValue(std::move(status_or_adapted).status()); + return ErrorValue::From(std::move(status_or_adapted).status(), arena); } return absl::visit( absl::Overload(OwningWellKnownTypesValueVisitor{ @@ -1421,7 +1442,7 @@ Value Value::WrapMessage( well_known_types::AdaptFromMessage(arena, *message, descriptor_pool, message_factory, scratch); if (ABSL_PREDICT_FALSE(!adapted_value.ok())) { - return ErrorValue(std::move(adapted_value).status()); + return ErrorValue::From(std::move(adapted_value).status(), arena); } return absl::visit( absl::Overload(BorrowingWellKnownTypesValueVisitor{ @@ -1455,7 +1476,7 @@ Value Value::WrapMessageUnsafe( well_known_types::AdaptFromMessage(arena, *message, descriptor_pool, message_factory, scratch); if (ABSL_PREDICT_FALSE(!adapted_value.ok())) { - return ErrorValue(std::move(adapted_value).status()); + return ErrorValue::From(std::move(adapted_value).status(), arena); } return absl::visit( absl::Overload(BorrowingWellKnownTypesValueVisitor{ @@ -1519,9 +1540,10 @@ Value WrapFieldImpl( if (ABSL_PREDICT_FALSE(reflection == nullptr)) { // This only happens for special implementations of Message that // should not normally be used with CEL. - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("failed to get reflection for message type: ", - message->GetDescriptor()->full_name()))); + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "failed to get reflection for message type: ", + message->GetDescriptor()->full_name())), + arena); } if (field->is_map()) { if constexpr (Unsafe::value) { @@ -1631,9 +1653,11 @@ Value WrapFieldImpl( case google::protobuf::FieldDescriptor::TYPE_SINT64: return IntValue(reflection->GetInt64(*message, field)); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("unexpected protocol buffer message field type: ", - field->type_name()))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("unexpected protocol buffer message field type: ", + field->type_name())), + arena); } } @@ -1661,14 +1685,16 @@ Value WrapRepeatedFieldImpl( if (ABSL_PREDICT_FALSE(reflection == nullptr)) { // This only happens for special implementations of Message that // should not normally be used with CEL. - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("failed to get reflection for message type: ", - message->GetDescriptor()->full_name()))); + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "failed to get reflection for message type: ", + message->GetDescriptor()->full_name())), + arena); } const int size = reflection->FieldSize(*message, field); if (ABSL_PREDICT_FALSE(index < 0 || index >= size)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("index out of bounds: ", index))); + return ErrorValue::From(absl::InvalidArgumentError( + absl::StrCat("index out of bounds: ", index)), + arena); } switch (field->type()) { case google::protobuf::FieldDescriptor::TYPE_DOUBLE: @@ -1757,8 +1783,10 @@ Value WrapRepeatedFieldImpl( return Value::Enum(field->enum_type(), reflection->GetRepeatedEnumValue( *message, field, index)); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("unexpected message field type: ", field->type_name()))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat( + "unexpected message field type: ", field->type_name())), + arena); } } @@ -1837,8 +1865,10 @@ Value WrapMapFieldValueImpl( case google::protobuf::FieldDescriptor::TYPE_ENUM: return Value::Enum(field->enum_type(), value.GetEnumValue()); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("unexpected message field type: ", field->type_name()))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat( + "unexpected message field type: ", field->type_name())), + arena); } } diff --git a/common/value.h b/common/value.h index f6ce5de1a..c13dad36e 100644 --- a/common/value.h +++ b/common/value.h @@ -102,8 +102,11 @@ class Value final : private common_internal::ValueMixin { // enums, returns `cel::IntValue`. For closed enums, returns `cel::ErrorValue` // if the value is not present in the enum otherwise returns `cel::IntValue`. static Value Enum(const google::protobuf::EnumValueDescriptor* absl_nonnull value); + ABSL_DEPRECATED("Use overload which takes an arena pointer") static Value Enum(const google::protobuf::EnumDescriptor* absl_nonnull type, int32_t number); + static Value Enum(const google::protobuf::EnumDescriptor* absl_nonnull type, + int32_t number, google::protobuf::Arena* absl_nonnull arena); // SFINAE overload for generated protobuf enums which are not well-known. // Always returns `cel::IntValue`. diff --git a/common/values/custom_map_value.cc b/common/values/custom_map_value.cc index ecd04abfd..7efb1de03 100644 --- a/common/values/custom_map_value.cc +++ b/common/values/custom_map_value.cc @@ -426,7 +426,7 @@ absl::Status CustomMapValueInterface::ForEach( CEL_ASSIGN_OR_RETURN( bool found, Find(key, descriptor_pool, message_factory, arena, &value)); if (!found) { - value = ErrorValue(NoSuchKeyError(key)); + value = ErrorValue::From(NoSuchKeyError(key), arena); } CEL_ASSIGN_OR_RETURN(auto ok, callback(key, value)); if (!ok) { @@ -639,7 +639,7 @@ absl::Status CustomMapValue::Get( case ValueKind::kUnknown: break; default: - *result = ErrorValue(NoSuchKeyError(key)); + *result = ErrorValue::From(NoSuchKeyError(key), arena); break; } } @@ -671,7 +671,7 @@ absl::StatusOr CustomMapValue::Find( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); return false; } @@ -682,7 +682,7 @@ absl::StatusOr CustomMapValue::Find( auto status_or_found = content.interface->Find( key, descriptor_pool, message_factory, arena, result); if (!status_or_found.ok()) { - *result = ErrorValue(std::move(status_or_found).status()); + *result = ErrorValue::From(std::move(status_or_found).status(), arena); return false; } if (!*status_or_found) { @@ -695,7 +695,7 @@ absl::StatusOr CustomMapValue::Find( dispatcher_->find(dispatcher_, content_, key, descriptor_pool, message_factory, arena, result); if (!status_or_found.ok()) { - *result = ErrorValue(std::move(status_or_found).status()); + *result = ErrorValue::From(std::move(status_or_found).status(), arena); return false; } if (!*status_or_found) { @@ -730,7 +730,7 @@ absl::Status CustomMapValue::Has( case ValueKind::kString: break; default: - *result = ErrorValue(InvalidMapKeyTypeError(key.kind())); + *result = ErrorValue::From(InvalidMapKeyTypeError(key.kind()), arena); return absl::OkStatus(); } if (dispatcher_ == nullptr) { @@ -740,7 +740,7 @@ absl::Status CustomMapValue::Has( auto status_or_has = content.interface->Has(key, descriptor_pool, message_factory, arena); if (!status_or_has.ok()) { - *result = ErrorValue(std::move(status_or_has).status()); + *result = ErrorValue::From(std::move(status_or_has).status(), arena); return absl::OkStatus(); } *result = BoolValue(*status_or_has); @@ -749,7 +749,7 @@ absl::Status CustomMapValue::Has( auto status_or_has = dispatcher_->has( dispatcher_, content_, key, descriptor_pool, message_factory, arena); if (!status_or_has.ok()) { - *result = ErrorValue(std::move(status_or_has).status()); + *result = ErrorValue::From(std::move(status_or_has).status(), arena); return absl::OkStatus(); } *result = BoolValue(*status_or_has); @@ -814,7 +814,7 @@ absl::Status CustomMapValue::ForEach( dispatcher_->find(dispatcher_, content_, key, descriptor_pool, message_factory, arena, &value)); if (!found) { - value = ErrorValue(NoSuchKeyError(key)); + value = ErrorValue::From(NoSuchKeyError(key), arena); } CEL_ASSIGN_OR_RETURN(auto ok, callback(key, value)); if (!ok) { diff --git a/common/values/error_value.h b/common/values/error_value.h index 7008df7fa..ea96bc857 100644 --- a/common/values/error_value.h +++ b/common/values/error_value.h @@ -66,6 +66,12 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final arena, google::protobuf::Arena::Create(arena, std::move(value))); } + [[nodiscard]] + static ErrorValue WrapUnsafe(const absl::Status* absl_nonnull value) { + ABSL_DCHECK(!value->ok()) << "ErrorValue requires a non-OK absl::Status"; + return ErrorValue(nullptr, value); + } + ABSL_DEPRECATED("Use From") explicit ErrorValue(absl::Status value) : arena_(nullptr), status_ptr_(nullptr) { diff --git a/common/values/optional_value.cc b/common/values/optional_value.cc index 7c214b9cb..3f4aa88f5 100644 --- a/common/values/optional_value.cc +++ b/common/values/optional_value.cc @@ -18,12 +18,14 @@ #include "absl/base/attributes.h" #include "absl/base/casts.h" +#include "absl/base/no_destructor.h" #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" +#include "absl/types/source_location.h" #include "common/arena.h" #include "common/native_type.h" #include "common/type.h" @@ -141,8 +143,10 @@ bool OptionalValueHasNoValue(const OptionalValueDispatcher* absl_nonnull, void EmptyOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, CustomValueContent content, cel::Value* absl_nonnull result) { - *result = - ErrorValue(absl::FailedPreconditionError("optional.none() dereference")); + static const absl::NoDestructor error( + absl::FailedPreconditionError("optional.none() dereference", + absl::SourceLocation())); + *result = ErrorValue::WrapUnsafe(&*error); } void NullOptionalValueValue(const OptionalValueDispatcher* absl_nonnull, diff --git a/common/values/parsed_json_value.cc b/common/values/parsed_json_value.cc index 6b10bea40..24ba36645 100644 --- a/common/values/parsed_json_value.cc +++ b/common/values/parsed_json_value.cc @@ -95,8 +95,9 @@ Value ParsedJsonValue(const google::protobuf::Message* absl_nonnull message, return ParsedJsonMapValue(&reflection.GetStructValue(*message), MessageArenaOr(message, arena)); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("unexpected value kind case: ", kind_case))); + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "unexpected value kind case: ", kind_case)), + arena); } } diff --git a/common/values/parsed_message_value.cc b/common/values/parsed_message_value.cc index 03d2d461d..62e3adc40 100644 --- a/common/values/parsed_message_value.cc +++ b/common/values/parsed_message_value.cc @@ -289,7 +289,7 @@ class ParsedMessageValueQualifyState final private: void SetResultFromError(absl::Status status, cel::MemoryManagerRef) override { - result_ = ErrorValue(std::move(status)); + result_ = ErrorValue::From(std::move(status), arena_); } void SetResultFromBool(bool value) override { result_ = BoolValue(value); } diff --git a/eval/eval/BUILD b/eval/eval/BUILD index f6ce6e221..1b4de3631 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -254,7 +254,6 @@ cc_library( ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:expr", "//common:value", "//eval/internal:errors", "//internal:status_macros", @@ -1249,6 +1248,7 @@ cc_library( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", ], ) diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc index 5e741d805..81e9c0270 100644 --- a/eval/eval/comprehension_step.cc +++ b/eval/eval/comprehension_step.cc @@ -168,7 +168,8 @@ absl::Status ComprehensionDirectStep::Evaluate1(ExecutionFrameBase& frame, result = std::move(range); return absl::OkStatus(); default: - result = cel::ErrorValue(CreateNoMatchingOverloadError("")); + result = cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), frame.arena()); return absl::OkStatus(); } ABSL_DCHECK(range_iter != nullptr); @@ -262,8 +263,8 @@ absl::StatusOr ComprehensionDirectStep::Evaluate1Unknown( result = std::move(condition); return true; default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError("")); + result = cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), frame.arena()); return true; } @@ -308,8 +309,8 @@ absl::StatusOr ComprehensionDirectStep::Evaluate1Known( result = std::move(condition); return true; default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError("")); + result = cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), frame.arena()); return true; } @@ -353,7 +354,8 @@ absl::Status ComprehensionDirectStep::Evaluate2(ExecutionFrameBase& frame, result = std::move(range); return absl::OkStatus(); default: - result = cel::ErrorValue(CreateNoMatchingOverloadError("")); + result = cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), frame.arena()); return absl::OkStatus(); } ABSL_DCHECK(range_iter != nullptr); @@ -417,8 +419,8 @@ absl::Status ComprehensionDirectStep::Evaluate2(ExecutionFrameBase& frame, should_skip_result = true; goto finish; default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError("")); + result = cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), frame.arena()); should_skip_result = true; goto finish; } @@ -475,8 +477,8 @@ absl::Status ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { default: // Replace with an error and jump past // ComprehensionFinishStep. - frame->value_stack().PopAndPush( - cel::ErrorValue(CreateNoMatchingOverloadError(""))); + frame->value_stack().PopAndPush(cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), frame->arena())); return frame->JumpTo(error_jump_offset_); } @@ -613,8 +615,9 @@ absl::Status ComprehensionCondStep::Evaluate1(ExecutionFrame* frame) const { return frame->JumpTo(error_jump_offset_); default: frame->value_stack().PopAndPush( - 2, - cel::ErrorValue(CreateNoMatchingOverloadError(""))); + 2, cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), + frame->arena())); frame->comprehension_slots().ClearSlot(iter_slot_); frame->comprehension_slots().ClearSlot(accu_slot_); frame->iterator_stack().Pop(); @@ -647,8 +650,9 @@ absl::Status ComprehensionCondStep::Evaluate2(ExecutionFrame* frame) const { return frame->JumpTo(error_jump_offset_); default: frame->value_stack().PopAndPush( - 2, - cel::ErrorValue(CreateNoMatchingOverloadError(""))); + 2, cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), + frame->arena())); frame->comprehension_slots().ClearSlot(iter_slot_); frame->comprehension_slots().ClearSlot(iter2_slot_); frame->comprehension_slots().ClearSlot(accu_slot_); diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index 4cf4ebf4d..911dbe08f 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -102,7 +102,8 @@ void LookupInMap(const MapValue& cel_map, const Value& key, cel_map.Find(key, frame.descriptor_pool(), frame.message_factory(), frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup).status()); + result = + cel::ErrorValue::From(std::move(lookup).status(), frame.arena()); return; } if (*lookup) { @@ -116,7 +117,8 @@ void LookupInMap(const MapValue& cel_map, const Value& key, cel_map.Find(IntValue(number->AsInt()), frame.descriptor_pool(), frame.message_factory(), frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup).status()); + result = + cel::ErrorValue::From(std::move(lookup).status(), frame.arena()); return; } if (*lookup) { @@ -130,7 +132,8 @@ void LookupInMap(const MapValue& cel_map, const Value& key, cel_map.Find(UintValue(number->AsUint()), frame.descriptor_pool(), frame.message_factory(), frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup).status()); + result = + cel::ErrorValue::From(std::move(lookup).status(), frame.arena()); return; } if (*lookup) { @@ -138,14 +141,15 @@ void LookupInMap(const MapValue& cel_map, const Value& key, return; } } - result = cel::ErrorValue(CreateNoSuchKeyError(key->DebugString())); + result = cel::ErrorValue::From(CreateNoSuchKeyError(key->DebugString()), + frame.arena()); return; } } absl::Status status = CheckMapKeyType(key); if (!status.ok()) { - result = cel::ErrorValue(std::move(status)); + result = cel::ErrorValue::From(std::move(status), frame.arena()); return; } @@ -153,7 +157,7 @@ void LookupInMap(const MapValue& cel_map, const Value& key, cel_map.Get(key, frame.descriptor_pool(), frame.message_factory(), frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup)); + result = cel::ErrorValue::From(std::move(lookup), frame.arena()); } ABSL_DCHECK(!result.IsUnknown()); } @@ -171,21 +175,25 @@ void LookupInList(const ListValue& cel_list, const Value& key, } if (!maybe_idx.has_value()) { - result = cel::ErrorValue(absl::UnknownError( - absl::StrCat("Index error: expected integer type, got ", - cel::KindToString(ValueKindToKind(key->kind()))))); + result = cel::ErrorValue::From( + absl::UnknownError( + absl::StrCat("Index error: expected integer type, got ", + cel::KindToString(ValueKindToKind(key->kind())))), + frame.arena()); return; } int64_t idx = *maybe_idx; auto size = cel_list.Size(); if (!size.ok()) { - result = cel::ErrorValue(size.status()); + result = cel::ErrorValue::From(size.status(), frame.arena()); return; } if (idx < 0 || idx >= *size) { - result = cel::ErrorValue(absl::UnknownError( - absl::StrCat("Index error: index=", idx, " size=", *size))); + result = + cel::ErrorValue::From(absl::UnknownError(absl::StrCat( + "Index error: index=", idx, " size=", *size)), + frame.arena()); return; } @@ -194,7 +202,7 @@ void LookupInList(const ListValue& cel_list, const Value& key, frame.arena(), &result); if (!lookup.ok()) { - result = cel::ErrorValue(std::move(lookup)); + result = cel::ErrorValue::From(std::move(lookup), frame.arena()); } ABSL_DCHECK(!result.IsUnknown()); } @@ -212,9 +220,11 @@ void LookupInContainer(const Value& container, const Value& key, return; } default: - result = cel::ErrorValue(absl::InvalidArgumentError( - absl::StrCat("Invalid container type: '", - ValueKindToString(container->kind()), "'"))); + result = + cel::ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "Invalid container type: '", + ValueKindToString(container->kind()), "'")), + frame.arena()); return; } } diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index 5d042baf5..e3c8365a4 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -92,8 +92,9 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( frame->type_provider().NewValueBuilder( name_, frame->message_factory(), frame->arena())); if (builder == nullptr) { - return ErrorValue( - absl::NotFoundError(absl::StrCat("Unable to find builder: ", name_))); + return ErrorValue::From( + absl::NotFoundError(absl::StrCat("Unable to find builder: ", name_)), + frame->arena()); } for (int i = 0; i < entries_size; ++i) { @@ -174,8 +175,9 @@ absl::Status DirectCreateStructStep::Evaluate(ExecutionFrameBase& frame, frame.type_provider().NewValueBuilder( name_, frame.message_factory(), frame.arena())); if (builder == nullptr) { - result = cel::ErrorValue( - absl::NotFoundError(absl::StrCat("Unable to find builder: ", name_))); + result = cel::ErrorValue::From( + absl::NotFoundError(absl::StrCat("Unable to find builder: ", name_)), + frame.arena()); return absl::OkStatus(); } diff --git a/eval/eval/equality_steps.cc b/eval/eval/equality_steps.cc index d720302e4..25bfe2e8a 100644 --- a/eval/eval/equality_steps.cc +++ b/eval/eval/equality_steps.cc @@ -69,8 +69,10 @@ absl::StatusOr EvaluateEquality( ValueEqualImpl(lhs, rhs, frame.descriptor_pool(), frame.message_factory(), frame.arena())); if (!is_equal.has_value()) { - return cel::ErrorValue(cel::runtime_internal::CreateNoMatchingOverloadError( - negation ? cel::builtin::kInequal : cel::builtin::kEqual)); + return cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + negation ? cel::builtin::kInequal : cel::builtin::kEqual), + frame.arena()); } return negation ? BoolValue(!*is_equal) : BoolValue(*is_equal); } @@ -140,9 +142,10 @@ absl::StatusOr EvaluateInMap(ExecutionFrameBase& frame, case ValueKind::kDouble: break; default: - return cel::ErrorValue( + return cel::ErrorValue::From( cel::runtime_internal::CreateNoMatchingOverloadError( - cel::builtin::kIn)); + cel::builtin::kIn), + frame.arena()); } Value result; CEL_RETURN_IF_ERROR(container.Has(item, frame.descriptor_pool(), @@ -210,8 +213,9 @@ absl::StatusOr EvaluateIn(ExecutionFrameBase& frame, const Value& item, if (container.IsMap()) { return EvaluateInMap(frame, item, container.GetMap()); } - return cel::ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(cel::builtin::kIn)); + return cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(cel::builtin::kIn), + frame.arena()); } class DirectInStep : public DirectExpressionStep { diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index 12c5af8a7..eb0f87dc3 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -229,16 +229,21 @@ Value NoOverloadResult(absl::string_view name, if (args.empty()) { // Should not be possible, but return a sensible error in case of logic // error. - return ErrorValue( - CreateNoMatchingOverloadError(absl::StrCat("().", name, "()"))); + return ErrorValue::From( + CreateNoMatchingOverloadError(absl::StrCat("().", name, "()")), + frame.arena()); } - return ErrorValue(CreateNoMatchingOverloadError(absl::StrCat( - "(", - ToLegacyKindName(cel::KindToString(ValueKindToKind(args[0].kind()))), - ").", name, CallArgTypeString(args.subspan(1))))); + return ErrorValue::From( + CreateNoMatchingOverloadError(absl::StrCat( + "(", + ToLegacyKindName( + cel::KindToString(ValueKindToKind(args[0].kind()))), + ").", name, CallArgTypeString(args.subspan(1)))), + frame.arena()); } - return cel::ErrorValue(CreateNoMatchingOverloadError( - absl::StrCat(name, CallArgTypeString(args)))); + return cel::ErrorValue::From(CreateNoMatchingOverloadError( + absl::StrCat(name, CallArgTypeString(args))), + frame.arena()); } absl::StatusOr AbstractFunctionStep::DoEvaluate( diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index 7ec1a3031..93c3b1755 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -66,8 +66,10 @@ absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, return absl::OkStatus(); } - result = cel::ErrorValue(CreateError( - absl::StrCat("No value with name \"", name, "\" found in Activation"))); + result = cel::ErrorValue::From( + CreateError(absl::StrCat("No value with name \"", name, + "\" found in Activation")), + frame.arena()); return absl::OkStatus(); } diff --git a/eval/eval/jump_step.cc b/eval/eval/jump_step.cc index 243a02e8a..03306ac80 100644 --- a/eval/eval/jump_step.cc +++ b/eval/eval/jump_step.cc @@ -129,8 +129,8 @@ class BoolCheckJumpStep : public JumpStepBase { } // Neither bool, error, nor unknown set. - Value error_value = - cel::ErrorValue(CreateNoMatchingOverloadError("")); + Value error_value = cel::ErrorValue::From( + CreateNoMatchingOverloadError(""), frame->arena()); frame->value_stack().PopAndPush(std::move(error_value)); return Jump(frame); diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index ed0a95275..84a5a648f 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -77,8 +77,10 @@ absl::Status ReturnLogicResult(ExecutionFrameBase& frame, OpType op_type, // Otherwise, add a no overload error. attribute_trail = AttributeTrail(); - lhs_result = cel::ErrorValue(CreateNoMatchingOverloadError( - op_type == OpType::kOr ? cel::builtin::kOr : cel::builtin::kAnd)); + lhs_result = cel::ErrorValue::From( + CreateNoMatchingOverloadError( + op_type == OpType::kOr ? cel::builtin::kOr : cel::builtin::kAnd), + frame.arena()); return absl::OkStatus(); } @@ -243,8 +245,11 @@ class LogicalOpStep : public ExpressionStepBase { result = args[error_pos.value()]; if (!result.IsError()) { - result = cel::ErrorValue(CreateNoMatchingOverloadError( - (op_type_ == OpType::kOr) ? cel::builtin::kOr : cel::builtin::kAnd)); + result = cel::ErrorValue::From( + CreateNoMatchingOverloadError((op_type_ == OpType::kOr) + ? cel::builtin::kOr + : cel::builtin::kAnd), + frame->arena()); } } @@ -314,8 +319,8 @@ absl::Status DirectNotStep::Evaluate(ExecutionFrameBase& frame, Value& result, // just forward. break; default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError(cel::builtin::kNot)); + result = cel::ErrorValue::From( + CreateNoMatchingOverloadError(cel::builtin::kNot), frame.arena()); break; } @@ -356,8 +361,8 @@ absl::Status IterativeNotStep::Evaluate(ExecutionFrame* frame) const { // just forward. break; default: - frame->value_stack().PopAndPush( - cel::ErrorValue(CreateNoMatchingOverloadError(cel::builtin::kNot))); + frame->value_stack().PopAndPush(cel::ErrorValue::From( + CreateNoMatchingOverloadError(cel::builtin::kNot), frame->arena())); break; } @@ -390,8 +395,8 @@ absl::Status DirectNotStrictlyFalseStep::Evaluate( result = BoolValue(true); break; default: - result = - cel::ErrorValue(CreateNoMatchingOverloadError(cel::builtin::kNot)); + result = cel::ErrorValue::From( + CreateNoMatchingOverloadError(cel::builtin::kNot), frame.arena()); break; } @@ -422,8 +427,8 @@ absl::Status IterativeNotStrictlyFalseStep::Evaluate( frame->value_stack().PopAndPush(BoolValue(true)); break; default: - frame->value_stack().PopAndPush( - cel::ErrorValue(CreateNoMatchingOverloadError(cel::builtin::kNot))); + frame->value_stack().PopAndPush(cel::ErrorValue::From( + CreateNoMatchingOverloadError(cel::builtin::kNot), frame->arena())); break; } diff --git a/eval/eval/optional_or_step.cc b/eval/eval/optional_or_step.cc index 1c52d91b6..6e474e404 100644 --- a/eval/eval/optional_or_step.cc +++ b/eval/eval/optional_or_step.cc @@ -32,6 +32,7 @@ #include "eval/eval/jump_step.h" #include "internal/status_macros.h" #include "runtime/internal/errors.h" +#include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -47,12 +48,12 @@ using ::cel::runtime_internal::CreateNoMatchingOverloadError; enum class OptionalOrKind { kOrOptional, kOrValue }; -ErrorValue MakeNoOverloadError(OptionalOrKind kind) { +ErrorValue MakeNoOverloadError(OptionalOrKind kind, google::protobuf::Arena* arena) { switch (kind) { case OptionalOrKind::kOrOptional: - return ErrorValue(CreateNoMatchingOverloadError("or")); + return ErrorValue::From(CreateNoMatchingOverloadError("or"), arena); case OptionalOrKind::kOrValue: - return ErrorValue(CreateNoMatchingOverloadError("orValue")); + return ErrorValue::From(CreateNoMatchingOverloadError("orValue"), arena); } ABSL_UNREACHABLE(); @@ -126,7 +127,7 @@ class OptionalOrStep : public ExpressionStepBase { absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, const Value& rhs, const AttributeTrail& lhs_attr, const AttributeTrail& rhs_attr, Value& result, - AttributeTrail& result_attr) { + AttributeTrail& result_attr, google::protobuf::Arena* arena) { if (InstanceOf(lhs) || InstanceOf(lhs)) { result = lhs; result_attr = lhs_attr; @@ -135,7 +136,7 @@ absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, auto lhs_optional_value = As(lhs); if (!lhs_optional_value.has_value()) { - result = MakeNoOverloadError(kind); + result = MakeNoOverloadError(kind, arena); result_attr = AttributeTrail(); return absl::OkStatus(); } @@ -152,7 +153,7 @@ absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, if (kind == OptionalOrKind::kOrOptional && !InstanceOf(rhs) && !InstanceOf(rhs) && !InstanceOf(rhs)) { - result = MakeNoOverloadError(kind); + result = MakeNoOverloadError(kind, arena); result_attr = AttributeTrail(); return absl::OkStatus(); } @@ -174,7 +175,8 @@ absl::Status OptionalOrStep::Evaluate(ExecutionFrame* frame) const { Value result; AttributeTrail result_attr; CEL_RETURN_IF_ERROR(EvalOptionalOr(kind_, args[0], args[1], args_attr[0], - args_attr[1], result, result_attr)); + args_attr[1], result, result_attr, + frame->arena())); frame->value_stack().PopAndPush(2, std::move(result), std::move(result_attr)); return absl::OkStatus(); @@ -207,7 +209,7 @@ absl::Status ExhaustiveDirectOptionalOrStep::Evaluate( AttributeTrail rhs_attr; CEL_RETURN_IF_ERROR(alternative_->Evaluate(frame, rhs, rhs_attr)); CEL_RETURN_IF_ERROR(EvalOptionalOr(kind_, result, rhs, attribute, rhs_attr, - result, attribute)); + result, attribute, frame.arena())); return absl::OkStatus(); } @@ -245,7 +247,7 @@ absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, auto optional_value = As(static_cast(result)); if (!optional_value.has_value()) { - result = MakeNoOverloadError(kind_); + result = MakeNoOverloadError(kind_, frame.arena()); return absl::OkStatus(); } @@ -264,7 +266,7 @@ absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, if (kind_ == OptionalOrKind::kOrOptional) { if (!InstanceOf(result) && !InstanceOf(result) && !InstanceOf(result)) { - result = MakeNoOverloadError(kind_); + result = MakeNoOverloadError(kind_, frame.arena()); } } diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index ec9e924aa..67054d75d 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -71,7 +71,7 @@ absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, // Log and return a CelError. ABSL_LOG(ERROR) << "Invalid attribute pattern matched select path: " << result.status().ToString(); // NOLINT: OSS compatibility - return cel::ErrorValue(std::move(result).status()); + return cel::ErrorValue::From(std::move(result).status(), frame.arena()); } return std::nullopt; @@ -119,7 +119,7 @@ absl::Status PerformHas(const Value& target, absl::string_view field, case ValueKind::kStruct: { auto has_field = target.GetStruct().HasFieldByName(field); if (!has_field.ok()) { - result = ErrorValue(std::move(has_field).status()); + result = ErrorValue::From(std::move(has_field).status(), arena); } else { result = BoolValue{*has_field}; } @@ -143,7 +143,7 @@ absl::Status PerformGet(const Value& target, absl::string_view field, auto status = target.GetMap().Get(field_value, descriptor_pool, message_factory, arena, &result); if (!status.ok()) { - result = ErrorValue(std::move(status)); + result = ErrorValue::From(std::move(status), arena); } return absl::OkStatus(); } @@ -152,7 +152,7 @@ absl::Status PerformGet(const Value& target, absl::string_view field, target, field, unboxing_option, descriptor_pool, message_factory, arena, enable_use_new_field_select_implementation, &result); if (!status.ok()) { - result = ErrorValue(std::move(status)); + result = ErrorValue::From(std::move(status), arena); } return absl::OkStatus(); } @@ -255,8 +255,9 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { } if (!(optional_arg || arg.IsMap() || arg.IsStruct())) { - frame->value_stack().PopAndPush(cel::ErrorValue(InvalidSelectTargetError()), - std::move(result_trail)); + frame->value_stack().PopAndPush( + cel::ErrorValue::From(InvalidSelectTargetError(), frame->arena()), + std::move(result_trail)); return absl::OkStatus(); } @@ -300,7 +301,7 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { frame->message_factory(), frame->arena(), frame->options().enable_use_new_field_select_implementation, result); if (!status.ok()) { - result = ErrorValue(std::move(status)); + result = ErrorValue::From(std::move(status), frame->arena()); } frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); return absl::OkStatus(); @@ -363,7 +364,8 @@ class DirectSelectStep : public DirectExpressionStep { if (optional_arg) { break; } - result = cel::ErrorValue(InvalidSelectTargetError()); + result = + cel::ErrorValue::From(InvalidSelectTargetError(), frame.arena()); return absl::OkStatus(); } @@ -394,7 +396,7 @@ class DirectSelectStep : public DirectExpressionStep { frame.descriptor_pool(), frame.message_factory(), frame.arena(), frame.options().enable_use_new_field_select_implementation, result); if (!status.ok()) { - result = ErrorValue(std::move(status)); + result = ErrorValue::From(std::move(status), frame.arena()); } return absl::OkStatus(); } diff --git a/eval/eval/ternary_step.cc b/eval/eval/ternary_step.cc index a12d6863e..ea543ab70 100644 --- a/eval/eval/ternary_step.cc +++ b/eval/eval/ternary_step.cc @@ -59,7 +59,8 @@ class ExhaustiveDirectTernaryStep : public DirectExpressionStep { } if (!condition.IsBool()) { - result = cel::ErrorValue(CreateNoMatchingOverloadError(kTernary)); + result = cel::ErrorValue::From(CreateNoMatchingOverloadError(kTernary), + frame.arena()); return absl::OkStatus(); } @@ -105,7 +106,8 @@ class ShortcircuitingDirectTernaryStep : public DirectExpressionStep { } if (!condition.IsBool()) { - result = cel::ErrorValue(CreateNoMatchingOverloadError(kTernary)); + result = cel::ErrorValue::From(CreateNoMatchingOverloadError(kTernary), + frame.arena()); return absl::OkStatus(); } @@ -157,7 +159,8 @@ absl::Status TernaryStep::Evaluate(ExecutionFrame* frame) const { cel::Value result; if (!condition.IsBool()) { - result = cel::ErrorValue(CreateNoMatchingOverloadError(kTernary)); + result = cel::ErrorValue::From(CreateNoMatchingOverloadError(kTernary), + frame->arena()); } else if (condition.GetBool().NativeValue()) { result = args[kTernaryStepTrue]; } else { diff --git a/extensions/BUILD b/extensions/BUILD index df5477112..743d32a9b 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -82,6 +82,7 @@ cc_library( "//eval/public:cel_number", "//eval/public:cel_options", "//internal:status_macros", + "//runtime:function", "//runtime:function_adapter", "//runtime:function_registry", "//runtime:runtime_options", @@ -591,6 +592,7 @@ cc_library( "//eval/public:cel_function_registry", "//eval/public:cel_options", "//internal:status_macros", + "//runtime:function", "//runtime:function_adapter", "//runtime:function_registry", "//runtime:runtime_options", @@ -746,7 +748,6 @@ cc_library( "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:btree", - "@com_google_absl//absl/memory", "@com_google_absl//absl/numeric:bits", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/extensions/encoders.cc b/extensions/encoders.cc index 66431b30b..12ba804e8 100644 --- a/extensions/encoders.cc +++ b/extensions/encoders.cc @@ -48,7 +48,8 @@ absl::StatusOr Base64Decode( std::string in; std::string out; if (!absl::Base64Unescape(value.NativeString(in), &out)) { - return ErrorValue{absl::InvalidArgumentError("invalid base64 data")}; + return ErrorValue::From(absl::InvalidArgumentError("invalid base64 data"), + arena); } return BytesValue(arena, std::move(out)); } diff --git a/extensions/formatting.cc b/extensions/formatting.cc index 252fdc7bd..c222be1ad 100644 --- a/extensions/formatting.cc +++ b/extensions/formatting.cc @@ -518,16 +518,17 @@ absl::StatusOr Format( } ++i; if (i >= format.size()) { - return ErrorValue( - absl::InvalidArgumentError("unexpected end of format string")); + return ErrorValue::From( + absl::InvalidArgumentError("unexpected end of format string"), arena); } if (format[i] == '%') { result.push_back('%'); continue; } if (arg_index >= args_size) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("index %d out of range", arg_index))); + return ErrorValue::From(absl::InvalidArgumentError(absl::StrFormat( + "index %d out of range", arg_index)), + arena); } CEL_ASSIGN_OR_RETURN(auto value, args.Get(arg_index++, descriptor_pool, message_factory, arena)); @@ -536,7 +537,7 @@ absl::StatusOr Format( descriptor_pool, message_factory, arena, clause_scratch); if (!clause.ok()) { - return ErrorValue(std::move(clause).status()); + return ErrorValue::From(std::move(clause).status(), arena); } absl::StrAppend(&result, clause->second); i += clause->first; diff --git a/extensions/lists_functions.cc b/extensions/lists_functions.cc index 7a40a0387..2617417d3 100644 --- a/extensions/lists_functions.cc +++ b/extensions/lists_functions.cc @@ -216,8 +216,9 @@ absl::StatusOr ListFlatten( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { if (depth < 0) { - return ErrorValue( - absl::InvalidArgumentError("flatten(): level must be non-negative")); + return ErrorValue::From( + absl::InvalidArgumentError("flatten(): level must be non-negative"), + arena); } auto builder = NewListValueBuilder(arena); CEL_RETURN_IF_ERROR(ListFlattenImpl(list, depth, descriptor_pool, @@ -231,13 +232,17 @@ absl::StatusOr ListRange( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { if (end < 0) { - return ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "lists.range: size must be non-negative, got %d", end))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "lists.range: size must be non-negative, got %d", end)), + arena); } if (end > max_range_size) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("lists.range: size %d exceeds maximum allowed (%d)", - end, max_range_size))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrFormat("lists.range: size %d exceeds maximum allowed (%d)", + end, max_range_size)), + arena); } auto builder = NewListValueBuilder(arena); builder->Reserve(end); @@ -269,18 +274,25 @@ absl::StatusOr ListSlice( google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(size_t size, list.Size()); if (start < 0 || end < 0) { - return ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "cannot slice(%d, %d), negative indexes not supported", start, end))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "cannot slice(%d, %d), negative indexes not supported", start, + end)), + arena); } if (start > end) { - return cel::ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("cannot slice(%d, %d), start index must be less than " - "or equal to end index", - start, end))); + return cel::ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "cannot slice(%d, %d), start index must be less than " + "or equal to end index", + start, end)), + arena); } if (size < end) { - return cel::ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "cannot slice(%d, %d), list is length %d", start, end, size))); + return cel::ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "cannot slice(%d, %d), list is length %d", start, end, size)), + arena); } auto builder = NewListValueBuilder(arena); for (int64_t i = start; i < end; ++i) { @@ -315,7 +327,7 @@ absl::StatusOr ListSortByAssociatedKeysNative( }, descriptor_pool, message_factory, arena); if (!status.ok()) { - return ErrorValue(status); + return ErrorValue::From(status, arena); } ABSL_ASSERT(keys_vec.size() == size); // Already checked by the caller. std::vector sorted_indices(keys_vec.size()); @@ -357,11 +369,13 @@ absl::StatusOr ListSortByAssociatedKeys( CEL_ASSIGN_OR_RETURN(size_t list_size, list.Size()); CEL_ASSIGN_OR_RETURN(size_t keys_size, keys.Size()); if (list_size != keys_size) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("@sortByAssociatedKeys() expected a list of the same " - "size as the associated keys list, but got %d and %d " - "elements respectively.", - list_size, keys_size))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "@sortByAssociatedKeys() expected a list of the same " + "size as the associated keys list, but got %d and %d " + "elements respectively.", + list_size, keys_size)), + arena); } // Empty lists are already sorted. // We don't check for size == 1 because the list could contain a single @@ -397,8 +411,10 @@ absl::StatusOr ListSortByAssociatedKeys( return ListSortByAssociatedKeysNative( list, keys, descriptor_pool, message_factory, arena); default: - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("sort(): unsupported type %s", first.GetTypeName()))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "sort(): unsupported type %s", first.GetTypeName())), + arena); } } diff --git a/extensions/math_ext.cc b/extensions/math_ext.cc index a7773da19..a31b112e8 100644 --- a/extensions/math_ext.cc +++ b/extensions/math_ext.cc @@ -31,6 +31,7 @@ #include "eval/public/cel_number.h" #include "eval/public/cel_options.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_adapter.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" @@ -102,15 +103,16 @@ absl::StatusOr MinList( google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(auto iterator, values.NewIterator()); if (!iterator->HasNext()) { - return ErrorValue( - absl::InvalidArgumentError("math.@min argument must not be empty")); + return ErrorValue::From( + absl::InvalidArgumentError("math.@min argument must not be empty"), + arena); } Value value; CEL_RETURN_IF_ERROR( iterator->Next(descriptor_pool, message_factory, arena, &value)); absl::StatusOr current = ValueToNumber(value, kMathMin); if (!current.ok()) { - return ErrorValue{current.status()}; + return ErrorValue::From(current.status(), arena); } CelNumber min = *current; while (iterator->HasNext()) { @@ -118,7 +120,7 @@ absl::StatusOr MinList( iterator->Next(descriptor_pool, message_factory, arena, &value)); absl::StatusOr other = ValueToNumber(value, kMathMin); if (!other.ok()) { - return ErrorValue{other.status()}; + return ErrorValue::From(other.status(), arena); } min = MinNumber(min, *other); } @@ -148,8 +150,9 @@ absl::StatusOr MaxList( google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(auto iterator, values.NewIterator()); if (!iterator->HasNext()) { - return ErrorValue( - absl::InvalidArgumentError("math.@max argument must not be empty")); + return ErrorValue::From( + absl::InvalidArgumentError("math.@max argument must not be empty"), + arena); } Value value; CEL_RETURN_IF_ERROR( @@ -219,9 +222,10 @@ bool IsFiniteDouble(double value) { return std::isfinite(value); } double AbsDouble(double value) { return std::fabs(value); } -Value AbsInt(int64_t value) { +Value AbsInt(int64_t value, const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(value == std::numeric_limits::min())) { - return ErrorValue(absl::InvalidArgumentError("integer overflow")); + return ErrorValue::From(absl::InvalidArgumentError("integer overflow"), + context.arena()); } return IntValue(value < 0 ? -value : value); } @@ -258,10 +262,13 @@ int64_t BitNotInt(int64_t value) { return ~value; } uint64_t BitNotUint(uint64_t value) { return ~value; } -Value BitShiftLeftInt(int64_t lhs, int64_t rhs) { +Value BitShiftLeftInt(int64_t lhs, int64_t rhs, + const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(rhs < 0)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("math.bitShiftLeft() invalid negative shift: ", rhs))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("math.bitShiftLeft() invalid negative shift: ", rhs)), + context.arena()); } if (rhs > 63) { return IntValue(0); @@ -273,10 +280,13 @@ Value BitShiftLeftInt(int64_t lhs, int64_t rhs) { << static_cast(rhs))); } -Value BitShiftLeftUint(uint64_t lhs, int64_t rhs) { +Value BitShiftLeftUint(uint64_t lhs, int64_t rhs, + const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(rhs < 0)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("math.bitShiftLeft() invalid negative shift: ", rhs))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("math.bitShiftLeft() invalid negative shift: ", rhs)), + context.arena()); } if (rhs > 63) { return UintValue(0); @@ -284,10 +294,13 @@ Value BitShiftLeftUint(uint64_t lhs, int64_t rhs) { return UintValue(lhs << static_cast(rhs)); } -Value BitShiftRightInt(int64_t lhs, int64_t rhs) { +Value BitShiftRightInt(int64_t lhs, int64_t rhs, + const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(rhs < 0)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("math.bitShiftRight() invalid negative shift: ", rhs))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("math.bitShiftRight() invalid negative shift: ", rhs)), + context.arena()); } if (rhs > 63) { return IntValue(0); @@ -298,10 +311,13 @@ Value BitShiftRightInt(int64_t lhs, int64_t rhs) { static_cast(rhs))); } -Value BitShiftRightUint(uint64_t lhs, int64_t rhs) { +Value BitShiftRightUint(uint64_t lhs, int64_t rhs, + const Function::InvokeContext& context) { if (ABSL_PREDICT_FALSE(rhs < 0)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrCat("math.bitShiftRight() invalid negative shift: ", rhs))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("math.bitShiftRight() invalid negative shift: ", rhs)), + context.arena()); } if (rhs > 63) { return UintValue(0); diff --git a/extensions/regex_ext.cc b/extensions/regex_ext.cc index 9c06d90c2..63bebe7cd 100644 --- a/extensions/regex_ext.cc +++ b/extensions/regex_ext.cc @@ -68,9 +68,11 @@ Value Extract(int regex_max_program_size, const StringValue& target, .With(ErrorValueReturn()); const int group_count = re2.NumberOfCapturingGroups(); if (group_count > 1) { - return ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "regular expression has more than one capturing group: %s", - regex_view))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "regular expression has more than one capturing group: %s", + regex_view)), + arena); } // Space for the full match (\0) and the first capture group (\1). @@ -100,9 +102,11 @@ Value ExtractAll(int regex_max_program_size, const StringValue& target, .With(ErrorValueReturn()); const int group_count = re2.NumberOfCapturingGroups(); if (group_count > 1) { - return ErrorValue(absl::InvalidArgumentError(absl::StrFormat( - "regular expression has more than one capturing group: %s", - regex_view))); + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrFormat( + "regular expression has more than one capturing group: %s", + regex_view)), + arena); } auto builder = NewListValueBuilder(arena); @@ -135,7 +139,7 @@ Value ExtractAll(int regex_max_program_size, const StringValue& target, absl::Status status = builder->Add(StringValue::From(desired_capture, arena)); if (!status.ok()) { - return ErrorValue(status); + return ErrorValue::From(status, arena); } temp_target.remove_prefix(full_match.data() - temp_target.data() + full_match.length()); @@ -161,8 +165,10 @@ Value ReplaceAll(int regex_max_program_size, const StringValue& target, .With(ErrorValueReturn()); std::string error_string; if (!re2.CheckRewriteString(replacement_view, &error_string)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("invalid replacement string: %s", error_string))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrFormat("invalid replacement string: %s", error_string)), + arena); } std::string output(target_view); @@ -197,8 +203,10 @@ Value ReplaceN(int regex_max_program_size, const StringValue& target, .With(ErrorValueReturn()); std::string error_string; if (!re2.CheckRewriteString(replacement_view, &error_string)) { - return ErrorValue(absl::InvalidArgumentError( - absl::StrFormat("invalid replacement string: %s", error_string))); + return ErrorValue::From( + absl::InvalidArgumentError( + absl::StrFormat("invalid replacement string: %s", error_string)), + arena); } std::string output; @@ -217,7 +225,8 @@ Value ReplaceN(int regex_max_program_size, const StringValue& target, if (!re2.Rewrite(&output, replacement_view, match, nmatch)) { // This should ideally not happen given CheckRewriteString passed - return ErrorValue(absl::InternalError("rewrite failed unexpectedly")); + return ErrorValue::From( + absl::InternalError("rewrite failed unexpectedly"), arena); } temp_target.remove_prefix(full_match.data() - temp_target.data() + diff --git a/extensions/regex_functions.cc b/extensions/regex_functions.cc index 005987ae4..249ecc563 100644 --- a/extensions/regex_functions.cc +++ b/extensions/regex_functions.cc @@ -69,8 +69,9 @@ Value ExtractString(int regex_max_program_size, const StringValue& target, std::string output; bool result = RE2::Extract(target_view, re2, rewrite_view, &output); if (!result) { - return ErrorValue(absl::InvalidArgumentError( - "Unable to extract string for the given regex")); + return ErrorValue::From(absl::InvalidArgumentError( + "Unable to extract string for the given regex"), + arena); } return StringValue::From(std::move(output), arena); } @@ -92,8 +93,9 @@ Value CaptureString(int regex_max_program_size, const StringValue& target, std::string output; bool result = RE2::FullMatch(target_view, re2, &output); if (!result) { - return ErrorValue(absl::InvalidArgumentError( - "Unable to capture groups for the given regex")); + return ErrorValue::From(absl::InvalidArgumentError( + "Unable to capture groups for the given regex"), + arena); } else { return StringValue::From(std::move(output), arena); } @@ -119,8 +121,10 @@ absl::StatusOr CaptureStringN( const int capturing_groups_count = re2.NumberOfCapturingGroups(); const auto& named_capturing_groups_map = re2.CapturingGroupNames(); if (capturing_groups_count <= 0) { - return ErrorValue(absl::InvalidArgumentError( - "Capturing groups were not found in the given regex.")); + return ErrorValue::From( + absl::InvalidArgumentError( + "Capturing groups were not found in the given regex."), + arena); } std::vector captured_strings(capturing_groups_count); std::vector captured_string_addresses(capturing_groups_count); @@ -132,8 +136,9 @@ absl::StatusOr CaptureStringN( bool result = RE2::FullMatchN(target_view, re2, argv.data(), capturing_groups_count); if (!result) { - return ErrorValue(absl::InvalidArgumentError( - "Unable to capture groups for the given regex")); + return ErrorValue::From(absl::InvalidArgumentError( + "Unable to capture groups for the given regex"), + arena); } auto builder = cel::NewMapValueBuilder(arena); builder->Reserve(capturing_groups_count); diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 4dcd7d594..937ca4737 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -343,9 +343,10 @@ absl::StatusOr ApplyQualifier( absl::Overload( [&](const FieldSpecifier& field_specifier) -> absl::StatusOr { if (!operand.Is()) { - return cel::ErrorValue( + return cel::ErrorValue::From( cel::runtime_internal::CreateNoMatchingOverloadError( - ""), + arena); } return WrappedStructGet(operand, field_specifier.name, descriptor_pool, message_factory, arena, @@ -355,21 +356,22 @@ absl::StatusOr ApplyQualifier( if (operand.Is()) { auto index_or = ListIndexFromQualifier(qualifier); if (!index_or.ok()) { - return cel::ErrorValue(index_or.status()); + return cel::ErrorValue::From(index_or.status(), arena); } return operand.GetList().Get(*index_or, descriptor_pool, message_factory, arena); } else if (operand.Is()) { auto key_or = MapKeyFromQualifier(qualifier, arena); if (!key_or.ok()) { - return cel::ErrorValue(key_or.status()); + return cel::ErrorValue::From(key_or.status(), arena); } return operand.GetMap().Get(*key_or, descriptor_pool, message_factory, arena); } - return cel::ErrorValue( + return cel::ErrorValue::From( cel::runtime_internal::CreateNoMatchingOverloadError( - cel::builtin::kIndex)); + cel::builtin::kIndex), + arena); }), qualifier); } @@ -403,9 +405,10 @@ absl::StatusOr FallbackSelect( [&](const FieldSpecifier& field_specifier) -> absl::StatusOr { if (!elem->Is()) { - return cel::ErrorValue( + return cel::ErrorValue::From( cel::runtime_internal::CreateNoMatchingOverloadError( - ""), + arena); } CEL_ASSIGN_OR_RETURN( bool present, @@ -414,9 +417,9 @@ absl::StatusOr FallbackSelect( }, [&](const AttributeQualifier& qualifier) -> absl::StatusOr { if (!elem->Is() || qualifier.kind() != Kind::kString) { - return cel::ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError( - "has")); + return cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError("has"), + arena); } return elem->GetMap().Has( diff --git a/extensions/strings.cc b/extensions/strings.cc index 54fda20d6..ad3719113 100644 --- a/extensions/strings.cc +++ b/extensions/strings.cc @@ -34,6 +34,7 @@ #include "eval/public/cel_options.h" #include "extensions/formatting.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_adapter.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" @@ -116,10 +117,11 @@ int64_t IndexOf2(const StringValue& haystack, const StringValue& needle) { } Value IndexOf3(const StringValue& haystack, const StringValue& needle, - int64_t pos) { + int64_t pos, const Function::InvokeContext& context) { if (pos > haystack.Size()) { - return ErrorValue{ - absl::InvalidArgumentError(absl::StrCat("index out of range: ", pos))}; + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat("index out of range: ", pos)), + context.arena()); } return IntValue(haystack.IndexOf(needle, pos).value_or(-1)); } @@ -129,10 +131,11 @@ int64_t LastIndexOf2(const StringValue& haystack, const StringValue& needle) { } Value LastIndexOf3(const StringValue& haystack, const StringValue& needle, - int64_t pos) { + int64_t pos, const Function::InvokeContext& context) { if (pos < 0 || pos > haystack.Size()) { - return ErrorValue{ - absl::InvalidArgumentError(absl::StrCat("index out of range: ", pos))}; + return ErrorValue::From( + absl::InvalidArgumentError(absl::StrCat("index out of range: ", pos)), + context.arena()); } return IntValue(haystack.LastIndexOf(needle, pos).value_or(-1)); } diff --git a/runtime/BUILD b/runtime/BUILD index 6831aeaae..57bdb1a63 100644 --- a/runtime/BUILD +++ b/runtime/BUILD @@ -575,6 +575,7 @@ cc_library( srcs = ["optional_types.cc"], hdrs = ["optional_types.h"], deps = [ + ":function", ":function_registry", ":runtime_builder", ":runtime_options", diff --git a/runtime/optional_types.cc b/runtime/optional_types.cc index 6678a05ed..6e5dbc12b 100644 --- a/runtime/optional_types.cc +++ b/runtime/optional_types.cc @@ -33,6 +33,7 @@ #include "internal/casts.h" #include "internal/number.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/internal/errors.h" #include "runtime/internal/runtime_friend_access.h" @@ -66,19 +67,24 @@ Value OptionalOfNonZeroValue( return OptionalOf(value, descriptor_pool, message_factory, arena); } -absl::StatusOr OptionalGetValue(const OpaqueValue& opaque_value) { +absl::StatusOr OptionalGetValue(const OpaqueValue& opaque_value, + const Function::InvokeContext& context) { if (auto optional_value = opaque_value.AsOptional(); optional_value) { return optional_value->Value(); } - return ErrorValue{runtime_internal::CreateNoMatchingOverloadError("value")}; + return ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("value"), + context.arena()); } -absl::StatusOr OptionalHasValue(const OpaqueValue& opaque_value) { +absl::StatusOr OptionalHasValue(const OpaqueValue& opaque_value, + const Function::InvokeContext& context) { if (auto optional_value = opaque_value.AsOptional(); optional_value) { return BoolValue{optional_value->HasValue()}; } - return ErrorValue{ - runtime_internal::CreateNoMatchingOverloadError("hasValue")}; + return ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("hasValue"), + context.arena()); } absl::StatusOr SelectOptionalFieldStruct( @@ -132,7 +138,8 @@ absl::StatusOr SelectOptionalField( message_factory, arena); } } - return ErrorValue{runtime_internal::CreateNoMatchingOverloadError("_[?_]")}; + return ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("_[?_]"), arena); } absl::StatusOr MapOptIndexOptionalValue( @@ -226,7 +233,8 @@ absl::StatusOr OptionalOptIndexOptionalValue( } } } - return ErrorValue{runtime_internal::CreateNoMatchingOverloadError("_[?_]")}; + return ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("_[?_]"), arena); } absl::StatusOr ListFirst(const cel::ListValue& list, @@ -281,7 +289,7 @@ absl::StatusOr ListUnwrapOpt( }, descriptor_pool, message_factory, arena); if (!status.ok()) { - return ErrorValue(status); + return ErrorValue::From(status, arena); } return std::move(*builder).Build(); } diff --git a/runtime/standard/BUILD b/runtime/standard/BUILD index 02a23ef1b..36eb2152b 100644 --- a/runtime/standard/BUILD +++ b/runtime/standard/BUILD @@ -159,6 +159,7 @@ cc_library( "//base:function_adapter", "//common:value", "//internal:status_macros", + "//runtime:function", "//runtime:function_registry", "//runtime:register_function_helper", "//runtime:runtime_options", @@ -276,6 +277,7 @@ cc_library( "//common:value", "//internal:overflow", "//internal:status_macros", + "//runtime:function", "//runtime:function_registry", "//runtime:runtime_options", "@com_google_absl//absl/status", @@ -307,12 +309,14 @@ cc_library( "//common:value", "//internal:overflow", "//internal:status_macros", + "//runtime:function", "//runtime:function_registry", "//runtime:runtime_options", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_protobuf//:protobuf", ], ) diff --git a/runtime/standard/arithmetic_functions.cc b/runtime/standard/arithmetic_functions.cc index a851ceb39..c77dd1a12 100644 --- a/runtime/standard/arithmetic_functions.cc +++ b/runtime/standard/arithmetic_functions.cc @@ -24,6 +24,7 @@ #include "common/value.h" #include "internal/overflow.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" @@ -32,93 +33,100 @@ namespace { // Template functions providing arithmetic operations template -Value Add(Type v0, Type v1); +Value Add(Type v0, Type v1, const Function::InvokeContext& context); template <> -Value Add(int64_t v0, int64_t v1) { +Value Add(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto sum = cel::internal::CheckedAdd(v0, v1); if (!sum.ok()) { - return ErrorValue(sum.status()); + return ErrorValue::From(sum.status(), context.arena()); } return IntValue(*sum); } template <> -Value Add(uint64_t v0, uint64_t v1) { +Value Add(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto sum = cel::internal::CheckedAdd(v0, v1); if (!sum.ok()) { - return ErrorValue(sum.status()); + return ErrorValue::From(sum.status(), context.arena()); } return UintValue(*sum); } template <> -Value Add(double v0, double v1) { +Value Add(double v0, double v1, const Function::InvokeContext&) { return DoubleValue(v0 + v1); } template -Value Sub(Type v0, Type v1); +Value Sub(Type v0, Type v1, const Function::InvokeContext& context); template <> -Value Sub(int64_t v0, int64_t v1) { +Value Sub(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto diff = cel::internal::CheckedSub(v0, v1); if (!diff.ok()) { - return ErrorValue(diff.status()); + return ErrorValue::From(diff.status(), context.arena()); } return IntValue(*diff); } template <> -Value Sub(uint64_t v0, uint64_t v1) { +Value Sub(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto diff = cel::internal::CheckedSub(v0, v1); if (!diff.ok()) { - return ErrorValue(diff.status()); + return ErrorValue::From(diff.status(), context.arena()); } return UintValue(*diff); } template <> -Value Sub(double v0, double v1) { +Value Sub(double v0, double v1, const Function::InvokeContext&) { return DoubleValue(v0 - v1); } template -Value Mul(Type v0, Type v1); +Value Mul(Type v0, Type v1, const Function::InvokeContext& context); template <> -Value Mul(int64_t v0, int64_t v1) { +Value Mul(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto prod = cel::internal::CheckedMul(v0, v1); if (!prod.ok()) { - return ErrorValue(prod.status()); + return ErrorValue::From(prod.status(), context.arena()); } return IntValue(*prod); } template <> -Value Mul(uint64_t v0, uint64_t v1) { +Value Mul(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto prod = cel::internal::CheckedMul(v0, v1); if (!prod.ok()) { - return ErrorValue(prod.status()); + return ErrorValue::From(prod.status(), context.arena()); } return UintValue(*prod); } template <> -Value Mul(double v0, double v1) { +Value Mul(double v0, double v1, const Function::InvokeContext&) { return DoubleValue(v0 * v1); } template -Value Div(Type v0, Type v1); +Value Div(Type v0, Type v1, const Function::InvokeContext& context); // Division operations for integer types should check for // division by 0 template <> -Value Div(int64_t v0, int64_t v1) { +Value Div(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto quot = cel::internal::CheckedDiv(v0, v1); if (!quot.ok()) { - return ErrorValue(quot.status()); + return ErrorValue::From(quot.status(), context.arena()); } return IntValue(*quot); } @@ -126,16 +134,17 @@ Value Div(int64_t v0, int64_t v1) { // Division operations for integer types should check for // division by 0 template <> -Value Div(uint64_t v0, uint64_t v1) { +Value Div(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto quot = cel::internal::CheckedDiv(v0, v1); if (!quot.ok()) { - return ErrorValue(quot.status()); + return ErrorValue::From(quot.status(), context.arena()); } return UintValue(*quot); } template <> -Value Div(double v0, double v1) { +Value Div(double v0, double v1, const Function::InvokeContext&) { static_assert(std::numeric_limits::is_iec559, "Division by zero for doubles must be supported"); @@ -145,24 +154,26 @@ Value Div(double v0, double v1) { // Modulo operation template -Value Modulo(Type v0, Type v1); +Value Modulo(Type v0, Type v1, const Function::InvokeContext& context); // Modulo operations for integer types should check for // division by 0 template <> -Value Modulo(int64_t v0, int64_t v1) { +Value Modulo(int64_t v0, int64_t v1, + const Function::InvokeContext& context) { auto mod = cel::internal::CheckedMod(v0, v1); if (!mod.ok()) { - return ErrorValue(mod.status()); + return ErrorValue::From(mod.status(), context.arena()); } return IntValue(*mod); } template <> -Value Modulo(uint64_t v0, uint64_t v1) { +Value Modulo(uint64_t v0, uint64_t v1, + const Function::InvokeContext& context) { auto mod = cel::internal::CheckedMod(v0, v1); if (!mod.ok()) { - return ErrorValue(mod.status()); + return ErrorValue::From(mod.status(), context.arena()); } return UintValue(*mod); } @@ -211,17 +222,17 @@ absl::Status RegisterArithmeticFunctions(FunctionRegistry& registry, &Modulo))); // Negation group - CEL_RETURN_IF_ERROR( - registry.Register(UnaryFunctionAdapter::CreateDescriptor( - cel::builtin::kNeg, false), - UnaryFunctionAdapter::WrapFunction( - [](int64_t value) -> Value { - auto inv = cel::internal::CheckedNegation(value); - if (!inv.ok()) { - return ErrorValue(inv.status()); - } - return IntValue(*inv); - }))); + CEL_RETURN_IF_ERROR(registry.Register( + UnaryFunctionAdapter::CreateDescriptor(cel::builtin::kNeg, + false), + UnaryFunctionAdapter::WrapFunction( + [](int64_t value, const Function::InvokeContext& context) -> Value { + auto inv = cel::internal::CheckedNegation(value); + if (!inv.ok()) { + return ErrorValue::From(inv.status(), context.arena()); + } + return IntValue(*inv); + }))); return registry.Register( UnaryFunctionAdapter::CreateDescriptor(cel::builtin::kNeg, diff --git a/runtime/standard/equality_functions.cc b/runtime/standard/equality_functions.cc index 4ff1acf9a..315413ea8 100644 --- a/runtime/standard/equality_functions.cc +++ b/runtime/standard/equality_functions.cc @@ -284,8 +284,8 @@ WrapComparison(Op op, absl::string_view name) { return BoolValue(*result); } - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(name)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(name), arena); }; } @@ -317,8 +317,8 @@ auto ComplexEquality(Op&& op) { CEL_ASSIGN_OR_RETURN(absl::optional result, op(t1, t2, descriptor_pool, message_factory, arena)); if (!result.has_value()) { - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(kEqual)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(kEqual), arena); } return BoolValue(*result); }; @@ -334,8 +334,9 @@ auto ComplexInequality(Op&& op) { CEL_ASSIGN_OR_RETURN(absl::optional result, op(t1, t2, descriptor_pool, message_factory, arena)); if (!result.has_value()) { - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(kInequal)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(kInequal), + arena); } return BoolValue(!*result); }; @@ -498,8 +499,8 @@ absl::StatusOr EqualOverloadImpl( if (result.has_value()) { return BoolValue(*result); } - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(kEqual)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(kEqual), arena); } absl::StatusOr InequalOverloadImpl( @@ -513,8 +514,8 @@ absl::StatusOr InequalOverloadImpl( if (result.has_value()) { return BoolValue(!*result); } - return ErrorValue( - cel::runtime_internal::CreateNoMatchingOverloadError(kInequal)); + return ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError(kInequal), arena); } absl::Status RegisterHeterogeneousEqualityFunctions( diff --git a/runtime/standard/logical_functions.cc b/runtime/standard/logical_functions.cc index cd3dd3cb5..6eed8b695 100644 --- a/runtime/standard/logical_functions.cc +++ b/runtime/standard/logical_functions.cc @@ -20,6 +20,7 @@ #include "base/function_adapter.h" #include "common/value.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/internal/errors.h" #include "runtime/register_function_helper.h" @@ -30,7 +31,8 @@ namespace { using ::cel::runtime_internal::CreateNoMatchingOverloadError; -Value NotStrictlyFalseImpl(const Value& value) { +Value NotStrictlyFalseImpl(const Value& value, + const Function::InvokeContext& context) { if (value.IsBool()) { return value; } @@ -40,7 +42,9 @@ Value NotStrictlyFalseImpl(const Value& value) { } // Should only accept bool unknown or error. - return ErrorValue(CreateNoMatchingOverloadError(builtin::kNotStrictlyFalse)); + return ErrorValue::From( + CreateNoMatchingOverloadError(builtin::kNotStrictlyFalse), + context.arena()); } } // namespace diff --git a/runtime/standard/time_functions.cc b/runtime/standard/time_functions.cc index a0ec5377c..d517e6f40 100644 --- a/runtime/standard/time_functions.cc +++ b/runtime/standard/time_functions.cc @@ -30,8 +30,10 @@ #include "common/value.h" #include "internal/overflow.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" +#include "google/protobuf/arena.h" namespace cel { namespace { @@ -73,55 +75,73 @@ absl::Status FindTimeBreakdown(absl::Time timestamp, absl::string_view tz, Value GetTimeBreakdownPart( absl::Time timestamp, absl::string_view tz, const std::function& - extractor_func) { + extractor_func, + google::protobuf::Arena* arena) { absl::TimeZone::CivilInfo breakdown; auto status = FindTimeBreakdown(timestamp, tz, &breakdown); if (!status.ok()) { - return ErrorValue(status); + return ErrorValue::From(status, arena); } return IntValue(extractor_func(breakdown)); } -Value GetFullYear(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.year(); - }); +Value GetFullYear(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.year(); + }, + context.arena()); } -Value GetMonth(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.month() - 1; - }); +Value GetMonth(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.month() - 1; + }, + context.arena()); } -Value GetDayOfYear(absl::Time timestamp, absl::string_view tz) { +Value GetDayOfYear(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { return GetTimeBreakdownPart( - timestamp, tz, [](const absl::TimeZone::CivilInfo& breakdown) { + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { return absl::GetYearDay(absl::CivilDay(breakdown.cs)) - 1; - }); + }, + context.arena()); } -Value GetDayOfMonth(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.day() - 1; - }); +Value GetDayOfMonth(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.day() - 1; + }, + context.arena()); } -Value GetDate(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.day(); - }); +Value GetDate(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.day(); + }, + context.arena()); } -Value GetDayOfWeek(absl::Time timestamp, absl::string_view tz) { +Value GetDayOfWeek(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { return GetTimeBreakdownPart( - timestamp, tz, [](const absl::TimeZone::CivilInfo& breakdown) { + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { absl::Weekday weekday = absl::GetWeekday(breakdown.cs); // get day of week from the date in UTC, zero-based, zero for Sunday, @@ -129,35 +149,48 @@ Value GetDayOfWeek(absl::Time timestamp, absl::string_view tz) { int weekday_num = static_cast(weekday); weekday_num = (weekday_num == 6) ? 0 : weekday_num + 1; return weekday_num; - }); + }, + context.arena()); } -Value GetHours(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.hour(); - }); +Value GetHours(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.hour(); + }, + context.arena()); } -Value GetMinutes(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.minute(); - }); +Value GetMinutes(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.minute(); + }, + context.arena()); } -Value GetSeconds(absl::Time timestamp, absl::string_view tz) { - return GetTimeBreakdownPart(timestamp, tz, - [](const absl::TimeZone::CivilInfo& breakdown) { - return breakdown.cs.second(); - }); +Value GetSeconds(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { + return GetTimeBreakdownPart( + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { + return breakdown.cs.second(); + }, + context.arena()); } -Value GetMilliseconds(absl::Time timestamp, absl::string_view tz) { +Value GetMilliseconds(absl::Time timestamp, absl::string_view tz, + const Function::InvokeContext& context) { return GetTimeBreakdownPart( - timestamp, tz, [](const absl::TimeZone::CivilInfo& breakdown) { + timestamp, tz, + [](const absl::TimeZone::CivilInfo& breakdown) { return absl::ToInt64Milliseconds(breakdown.subsecond); - }); + }, + context.arena()); } absl::Status RegisterTimestampFunctions(FunctionRegistry& registry, @@ -166,141 +199,171 @@ absl::Status RegisterTimestampFunctions(FunctionRegistry& registry, BinaryFunctionAdapter:: CreateDescriptor(builtin::kFullYear, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetFullYear(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetFullYear(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kFullYear, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetFullYear(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetFullYear(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kMonth, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetMonth(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetMonth(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor(builtin::kMonth, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetMonth(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetMonth(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kDayOfYear, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetDayOfYear(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetDayOfYear(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kDayOfYear, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetDayOfYear(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetDayOfYear(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kDayOfMonth, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetDayOfMonth(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetDayOfMonth(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kDayOfMonth, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetDayOfMonth(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetDayOfMonth(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kDate, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetDate(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetDate(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor(builtin::kDate, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetDate(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetDate(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kDayOfWeek, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetDayOfWeek(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetDayOfWeek(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kDayOfWeek, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetDayOfWeek(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetDayOfWeek(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kHours, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetHours(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetHours(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor(builtin::kHours, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetHours(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetHours(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kMinutes, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetMinutes(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetMinutes(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kMinutes, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetMinutes(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetMinutes(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kSeconds, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetSeconds(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetSeconds(ts, tz.ToString(), context); }))); CEL_RETURN_IF_ERROR(registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kSeconds, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetSeconds(ts, ""); }))); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetSeconds(ts, "", context); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter:: CreateDescriptor(builtin::kMilliseconds, true), BinaryFunctionAdapter:: - WrapFunction([](absl::Time ts, const StringValue& tz) -> Value { - return GetMilliseconds(ts, tz.ToString()); + WrapFunction([](absl::Time ts, const StringValue& tz, + const Function::InvokeContext& context) -> Value { + return GetMilliseconds(ts, tz.ToString(), context); }))); return registry.Register( UnaryFunctionAdapter::CreateDescriptor( builtin::kMilliseconds, true), UnaryFunctionAdapter::WrapFunction( - [](absl::Time ts) -> Value { return GetMilliseconds(ts, ""); })); + [](absl::Time ts, const Function::InvokeContext& context) -> Value { + return GetMilliseconds(ts, "", context); + })); } absl::Status RegisterCheckedTimeArithmeticFunctions( @@ -310,84 +373,90 @@ absl::Status RegisterCheckedTimeArithmeticFunctions( absl::Duration>::CreateDescriptor(builtin::kAdd, false), BinaryFunctionAdapter, absl::Time, absl::Duration>:: - WrapFunction( - [](absl::Time t1, absl::Duration d2) -> absl::StatusOr { - auto sum = cel::internal::CheckedAdd(t1, d2); - if (!sum.ok()) { - return ErrorValue(sum.status()); - } - return TimestampValue(*sum); - }))); + WrapFunction([](absl::Time t1, absl::Duration d2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto sum = cel::internal::CheckedAdd(t1, d2); + if (!sum.ok()) { + return ErrorValue::From(sum.status(), context.arena()); + } + return TimestampValue(*sum); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter, absl::Duration, absl::Time>::CreateDescriptor(builtin::kAdd, false), BinaryFunctionAdapter, absl::Duration, absl::Time>:: - WrapFunction( - [](absl::Duration d2, absl::Time t1) -> absl::StatusOr { - auto sum = cel::internal::CheckedAdd(t1, d2); - if (!sum.ok()) { - return ErrorValue(sum.status()); - } - return TimestampValue(*sum); - }))); + WrapFunction([](absl::Duration d2, absl::Time t1, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto sum = cel::internal::CheckedAdd(t1, d2); + if (!sum.ok()) { + return ErrorValue::From(sum.status(), context.arena()); + } + return TimestampValue(*sum); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter, absl::Duration, absl::Duration>::CreateDescriptor(builtin::kAdd, false), - BinaryFunctionAdapter< - absl::StatusOr, absl::Duration, - absl::Duration>::WrapFunction([](absl::Duration d1, absl::Duration d2) - -> absl::StatusOr { - auto sum = cel::internal::CheckedAdd(d1, d2); - if (!sum.ok()) { - return ErrorValue(sum.status()); - } - return DurationValue(*sum); - }))); + BinaryFunctionAdapter, absl::Duration, + absl::Duration>:: + WrapFunction([](absl::Duration d1, absl::Duration d2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto sum = cel::internal::CheckedAdd(d1, d2); + if (!sum.ok()) { + return ErrorValue::From(sum.status(), context.arena()); + } + return DurationValue(*sum); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter, absl::Time, absl::Duration>:: CreateDescriptor(builtin::kSubtract, false), BinaryFunctionAdapter, absl::Time, absl::Duration>:: - WrapFunction( - [](absl::Time t1, absl::Duration d2) -> absl::StatusOr { - auto diff = cel::internal::CheckedSub(t1, d2); - if (!diff.ok()) { - return ErrorValue(diff.status()); - } - return TimestampValue(*diff); - }))); + WrapFunction([](absl::Time t1, absl::Duration d2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto diff = cel::internal::CheckedSub(t1, d2); + if (!diff.ok()) { + return ErrorValue::From(diff.status(), context.arena()); + } + return TimestampValue(*diff); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter, absl::Time, absl::Time>::CreateDescriptor(builtin::kSubtract, false), BinaryFunctionAdapter, absl::Time, absl::Time>:: - WrapFunction( - [](absl::Time t1, absl::Time t2) -> absl::StatusOr { - auto diff = cel::internal::CheckedSub(t1, t2); - if (!diff.ok()) { - return ErrorValue(diff.status()); - } - return DurationValue(*diff); - }))); + WrapFunction([](absl::Time t1, absl::Time t2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto diff = cel::internal::CheckedSub(t1, t2); + if (!diff.ok()) { + return ErrorValue::From(diff.status(), context.arena()); + } + return DurationValue(*diff); + }))); CEL_RETURN_IF_ERROR(registry.Register( BinaryFunctionAdapter< absl::StatusOr, absl::Duration, absl::Duration>::CreateDescriptor(builtin::kSubtract, false), - BinaryFunctionAdapter< - absl::StatusOr, absl::Duration, - absl::Duration>::WrapFunction([](absl::Duration d1, absl::Duration d2) - -> absl::StatusOr { - auto diff = cel::internal::CheckedSub(d1, d2); - if (!diff.ok()) { - return ErrorValue(diff.status()); - } - return DurationValue(*diff); - }))); + BinaryFunctionAdapter, absl::Duration, + absl::Duration>:: + WrapFunction([](absl::Duration d1, absl::Duration d2, + const Function::InvokeContext& context) + -> absl::StatusOr { + auto diff = cel::internal::CheckedSub(d1, d2); + if (!diff.ok()) { + return ErrorValue::From(diff.status(), context.arena()); + } + return DurationValue(*diff); + }))); return absl::OkStatus(); } diff --git a/runtime/standard/type_conversion_functions.cc b/runtime/standard/type_conversion_functions.cc index 2400c8fdf..f99f07f31 100644 --- a/runtime/standard/type_conversion_functions.cc +++ b/runtime/standard/type_conversion_functions.cc @@ -23,7 +23,6 @@ #include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "base/builtins.h" @@ -66,8 +65,11 @@ Value FormatDouble(double v, const Function::InvokeContext& context) { std::to_chars_result result = std::to_chars(buf, buf + kBufSize, v, std::chars_format::general); if (result.ec != std::errc()) { - return cel::ErrorValue(absl::InvalidArgumentError(absl::StrCat( - "double format error: ", std::make_error_code(result.ec).message()))); + return cel::ErrorValue::From( + absl::InvalidArgumentError( + absl::StrCat("double format error: ", + std::make_error_code(result.ec).message())), + context.arena()); } absl::string_view out(buf, result.ptr - buf); return StringValue::From(out, arena); @@ -89,7 +91,8 @@ absl::Status RegisterBoolConversionFunctions(FunctionRegistry& registry, // string -> bool return UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kBool, - [](const StringValue& v) -> Value { + [](const StringValue& v, + const Function::InvokeContext& context) -> Value { if ((v == "true") || (v == "True") || (v == "TRUE") || (v == "t") || (v == "1")) { return TrueValue(); @@ -97,8 +100,10 @@ absl::Status RegisterBoolConversionFunctions(FunctionRegistry& registry, (v == "f") || (v == "0")) { return FalseValue(); } else { - return ErrorValue(absl::InvalidArgumentError( - "Type conversion error from 'string' to 'bool'")); + return ErrorValue::From( + absl::InvalidArgumentError( + "Type conversion error from 'string' to 'bool'"), + context.arena()); } }, registry); @@ -116,10 +121,10 @@ absl::Status RegisterIntConversionFunctions(FunctionRegistry& registry, // double -> int status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kInt, - [](double v) -> Value { + [](double v, const Function::InvokeContext& context) -> Value { auto conv = cel::internal::CheckedDoubleToInt64(v); if (!conv.ok()) { - return ErrorValue(conv.status()); + return ErrorValue::From(conv.status(), context.arena()); } return IntValue(*conv); }, @@ -135,11 +140,13 @@ absl::Status RegisterIntConversionFunctions(FunctionRegistry& registry, status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kInt, - [](const StringValue& s) -> Value { + [](const StringValue& s, + const Function::InvokeContext& context) -> Value { int64_t result; if (!absl::SimpleAtoi(s.ToString(), &result)) { - return ErrorValue( - absl::InvalidArgumentError("cannot convert string to int")); + return ErrorValue::From( + absl::InvalidArgumentError("cannot convert string to int"), + context.arena()); } return IntValue(result); }, @@ -155,10 +162,10 @@ absl::Status RegisterIntConversionFunctions(FunctionRegistry& registry, // uint -> int return UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kInt, - [](uint64_t v) -> Value { + [](uint64_t v, const Function::InvokeContext& context) -> Value { auto conv = cel::internal::CheckedUint64ToInt64(v); if (!conv.ok()) { - return ErrorValue(conv.status()); + return ErrorValue::From(conv.status(), context.arena()); } return IntValue(*conv); }, @@ -176,13 +183,15 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](const BytesValue& value) -> Value { + [](const BytesValue& value, + const Function::InvokeContext& context) -> Value { auto valid = value.NativeValue([](const auto& value) -> bool { return internal::Utf8IsValid(value); }); if (!valid) { - return ErrorValue( - absl::InvalidArgumentError("malformed UTF-8 bytes")); + return ErrorValue::From( + absl::InvalidArgumentError("malformed UTF-8 bytes"), + context.arena()); } return StringValue(value.ToString()); }, @@ -234,10 +243,11 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, // duration -> string status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](absl::Duration value) -> Value { + [](absl::Duration value, + const Function::InvokeContext& context) -> Value { auto encode = EncodeDurationToJson(value); if (!encode.ok()) { - return ErrorValue(encode.status()); + return ErrorValue::From(encode.status(), context.arena()); } return StringValue(*encode); }, @@ -247,10 +257,10 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, // timestamp -> string return UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](absl::Time value) -> Value { + [](absl::Time value, const Function::InvokeContext& context) -> Value { auto encode = EncodeTimestampToJson(value); if (!encode.ok()) { - return ErrorValue(encode.status()); + return ErrorValue::From(encode.status(), context.arena()); } return StringValue(*encode); }, @@ -263,10 +273,10 @@ absl::Status RegisterUintConversionFunctions(FunctionRegistry& registry, absl::Status status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kUint, - [](double v) -> Value { + [](double v, const Function::InvokeContext& context) -> Value { auto conv = cel::internal::CheckedDoubleToUint64(v); if (!conv.ok()) { - return ErrorValue(conv.status()); + return ErrorValue::From(conv.status(), context.arena()); } return UintValue(*conv); }, @@ -276,10 +286,10 @@ absl::Status RegisterUintConversionFunctions(FunctionRegistry& registry, // int -> uint status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kUint, - [](int64_t v) -> Value { + [](int64_t v, const Function::InvokeContext& context) -> Value { auto conv = cel::internal::CheckedInt64ToUint64(v); if (!conv.ok()) { - return ErrorValue(conv.status()); + return ErrorValue::From(conv.status(), context.arena()); } return UintValue(*conv); }, @@ -290,11 +300,13 @@ absl::Status RegisterUintConversionFunctions(FunctionRegistry& registry, status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kUint, - [](const StringValue& s) -> Value { + [](const StringValue& s, + const Function::InvokeContext& context) -> Value { uint64_t result; if (!absl::SimpleAtoi(s.ToString(), &result)) { - return ErrorValue( - absl::InvalidArgumentError("cannot convert string to uint")); + return ErrorValue::From( + absl::InvalidArgumentError("cannot convert string to uint"), + context.arena()); } return UintValue(result); }, @@ -342,13 +354,15 @@ absl::Status RegisterDoubleConversionFunctions(FunctionRegistry& registry, status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kDouble, - [](const StringValue& s) -> Value { + [](const StringValue& s, + const Function::InvokeContext& context) -> Value { double result; if (absl::SimpleAtod(s.ToString(), &result)) { return DoubleValue(result); } else { - return ErrorValue(absl::InvalidArgumentError( - "cannot convert string to double")); + return ErrorValue::From( + absl::InvalidArgumentError("cannot convert string to double"), + context.arena()); } }, registry); @@ -360,16 +374,18 @@ absl::Status RegisterDoubleConversionFunctions(FunctionRegistry& registry, registry); } -Value CreateDurationFromString(const StringValue& dur_str) { +Value CreateDurationFromString(const StringValue& dur_str, + const Function::InvokeContext& context) { absl::Duration d; if (!absl::ParseDuration(dur_str.ToString(), &d)) { - return ErrorValue( - absl::InvalidArgumentError("String to Duration conversion failed")); + return ErrorValue::From( + absl::InvalidArgumentError("String to Duration conversion failed"), + context.arena()); } auto status = internal::ValidateDuration(d); if (!status.ok()) { - return ErrorValue(std::move(status)); + return ErrorValue::From(std::move(status), context.arena()); } return DurationValue(d); } @@ -388,11 +404,14 @@ absl::Status RegisterTimeConversionFunctions(FunctionRegistry& registry, CEL_RETURN_IF_ERROR( (UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kTimestamp, - [=](int64_t epoch_seconds) -> Value { + [=](int64_t epoch_seconds, + const Function::InvokeContext& context) -> Value { absl::Time ts = absl::FromUnixSeconds(epoch_seconds); if (enable_timestamp_duration_overflow_errors) { if (ts < MinTimestamp() || ts > MaxTimestamp()) { - return ErrorValue(absl::OutOfRangeError("timestamp overflow")); + return ErrorValue::From( + absl::OutOfRangeError("timestamp overflow"), + context.arena()); } } return UnsafeTimestampValue(ts); @@ -419,16 +438,21 @@ absl::Status RegisterTimeConversionFunctions(FunctionRegistry& registry, return UnaryFunctionAdapter:: RegisterGlobalOverload( cel::builtin::kTimestamp, - [=](const StringValue& time_str) -> Value { + [=](const StringValue& time_str, + const Function::InvokeContext& context) -> Value { absl::Time ts; if (!absl::ParseTime(absl::RFC3339_full, time_str.ToString(), &ts, nullptr)) { - return ErrorValue(absl::InvalidArgumentError( - "String to Timestamp conversion failed")); + return ErrorValue::From( + absl::InvalidArgumentError( + "String to Timestamp conversion failed"), + context.arena()); } if (enable_timestamp_duration_overflow_errors) { if (ts < MinTimestamp() || ts > MaxTimestamp()) { - return ErrorValue(absl::OutOfRangeError("timestamp overflow")); + return ErrorValue::From( + absl::OutOfRangeError("timestamp overflow"), + context.arena()); } } return UnsafeTimestampValue(ts); From 49005a51b0a9131c67b9072d24ad8cb84cc201e7 Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 28 Sep 2026 13:36:50 -0700 Subject: [PATCH 241/350] Make `cel::BytesValue` and `cel::StringValue` trivial PiperOrigin-RevId: 989815840 --- common/BUILD | 1 - common/internal/BUILD | 10 +- common/internal/byte_string.cc | 780 +++++----------- common/internal/byte_string.h | 294 ++---- common/internal/byte_string_test.cc | 859 ++++-------------- common/internal/value_conversion.cc | 4 +- common/legacy_value.cc | 12 +- common/type_reflector_test.cc | 48 +- common/value.cc | 144 +-- common/value.h | 3 + common/value_testing_test.cc | 16 +- common/values/bytes_value.h | 145 +-- common/values/bytes_value_input_stream.h | 37 +- common/values/bytes_value_output_stream.h | 89 +- common/values/bytes_value_test.cc | 111 +-- common/values/custom_list_value.cc | 5 +- common/values/custom_map_value.cc | 5 +- common/values/custom_map_value_test.cc | 104 +-- common/values/legacy_struct_value_test.cc | 20 +- common/values/map_value_test.cc | 7 +- common/values/mutable_list_value_test.cc | 15 +- common/values/mutable_map_value_test.cc | 36 +- common/values/parsed_json_list_value_test.cc | 14 +- common/values/parsed_json_map_value.cc | 2 +- common/values/parsed_json_map_value_test.cc | 32 +- common/values/parsed_json_value.cc | 13 +- common/values/parsed_map_field_value_test.cc | 26 +- .../parsed_repeated_field_value_test.cc | 4 +- common/values/string_value.cc | 137 +-- common/values/string_value.h | 145 +-- common/values/string_value_test.cc | 356 ++++---- common/values/value_variant.cc | 247 ----- common/values/value_variant.h | 11 +- common/values/value_variant_test.cc | 8 +- conformance/policy/policy_conformance_test.cc | 10 +- eval/compiler/flat_expr_builder.cc | 4 +- eval/eval/comprehension_slots_test.cc | 4 +- eval/eval/function_step_test.cc | 2 +- eval/eval/select_step.cc | 55 +- eval/eval/select_step.h | 8 +- eval/eval/select_step_test.cc | 225 ++--- eval/tests/modern_benchmark_test.cc | 18 +- extensions/encoders.cc | 4 +- extensions/formatting_test.cc | 4 +- extensions/protobuf/value_test.cc | 5 +- extensions/select_optimization.cc | 2 +- extensions/select_optimization_test.cc | 4 +- runtime/constant_folding_test.cc | 42 +- runtime/function_adapter_test.cc | 127 +-- runtime/internal/BUILD | 4 +- runtime/internal/convert_constant.cc | 13 +- runtime/internal/convert_constant.h | 7 +- runtime/internal/function_adapter_test.cc | 8 +- runtime/regex_precompilation_test.cc | 20 +- runtime/standard/type_conversion_functions.cc | 20 +- 55 files changed, 1552 insertions(+), 2774 deletions(-) diff --git a/common/BUILD b/common/BUILD index ba9f2db54..ab1d389a8 100644 --- a/common/BUILD +++ b/common/BUILD @@ -789,7 +789,6 @@ cc_library( ":value_kind", "//base:attributes", "//common/internal:byte_string", - "//common/internal:reference_count", "//eval/internal:cel_value_equal", "//eval/public:cel_value", "//eval/public:message_wrapper", diff --git a/common/internal/BUILD b/common/internal/BUILD index 3be350754..eebf66219 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -67,11 +67,6 @@ cc_library( srcs = ["byte_string.cc"], hdrs = ["byte_string.h"], deps = [ - ":metadata", - ":reference_count", - "//common:allocator", - "//common:arena", - "//common:memory", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:overload", @@ -79,6 +74,7 @@ cc_library( "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", + "@com_google_absl//absl/strings:resize_and_overwrite", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", @@ -90,16 +86,12 @@ cc_test( srcs = ["byte_string_test.cc"], deps = [ ":byte_string", - ":reference_count", - "//common:allocator", - "//common:memory", "//internal:testing", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/hash", "@com_google_absl//absl/strings:cord", "@com_google_absl//absl/strings:cord_test_helpers", "@com_google_absl//absl/strings:string_view", - "@com_google_absl//absl/types:optional", "@com_google_protobuf//:protobuf", ], ) diff --git a/common/internal/byte_string.cc b/common/internal/byte_string.cc index 201842a10..468bf893a 100644 --- a/common/internal/byte_string.cc +++ b/common/internal/byte_string.cc @@ -14,11 +14,11 @@ #include "common/internal/byte_string.h" +#include #include #include #include #include -#include #include #include "absl/base/nullability.h" @@ -28,26 +28,62 @@ #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/match.h" +#include "absl/strings/resize_and_overwrite.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/internal/metadata.h" -#include "common/internal/reference_count.h" -#include "common/memory.h" #include "google/protobuf/arena.h" namespace cel::common_internal { namespace { -char* CopyCordToArray(const absl::Cord& cord, char* data) { +char* CopyCordToArray(const absl::Cord& cord, size_t offset, size_t size, + char* data) { for (auto chunk : cord.Chunks()) { - std::memcpy(data, chunk.data(), chunk.size()); - data += chunk.size(); + if (size == 0) { + break; + } + if (offset > 0) { + size_t min_offset = std::min(chunk.size(), offset); + offset -= min_offset; + if (offset > 0) { + continue; + } + chunk.remove_prefix(min_offset); + } + size_t min_size = std::min(size, chunk.size()); + std::memcpy(data, chunk.data(), min_size); + data += min_size; + size -= min_size; } return data; } +char* CopyCordToArray(const absl::Cord& cord, char* data) { + return (CopyCordToArray)(cord, 0, cord.size(), data); +} + +void AppendCordToString(const absl::Cord& cord, size_t offset, size_t size, + std::string& data) { + data.reserve(data.size() + size); + for (auto chunk : cord.Chunks()) { + if (size == 0) { + break; + } + if (offset > 0) { + size_t min_offset = std::min(chunk.size(), offset); + offset -= min_offset; + if (offset > 0) { + continue; + } + chunk.remove_prefix(min_offset); + } + size_t min_size = std::min(size, chunk.size()); + data.append(absl::string_view(chunk.data(), min_size)); + size -= min_size; + } +} + template T ConsumeAndDestroy(T& object) { T consumed = std::move(object); @@ -57,6 +93,80 @@ T ConsumeAndDestroy(T& object) { } // namespace +ByteString ByteString::From(const char* absl_nullable value, + google::protobuf::Arena* absl_nonnull arena) { + return From(absl::NullSafeStringView(value), arena); +} + +ByteString ByteString::From(absl::string_view value, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + ByteString result(UninitializedTag{}); + if (value.size() <= kSmallByteStringCapacity) { + result.SetSmall(arena, value); + } else { + char* arena_value = + reinterpret_cast(arena->AllocateAligned(value.size())); + std::memcpy(arena_value, value.data(), value.size()); + result.SetMedium(arena, absl::string_view(arena_value, value.size())); + } + return result; +} + +ByteString ByteString::From(const absl::Cord& value, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + ByteString result(UninitializedTag{}); + if (value.size() <= kSmallByteStringCapacity) { + result.SetSmall(arena, value); + } else { + result.SetLarge(arena, google::protobuf::Arena::Create(arena, value)); + } + return result; +} + +ByteString ByteString::From(std::string&& value, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + ByteString result(UninitializedTag{}); + if (value.size() <= kSmallByteStringCapacity) { + result.SetSmall(arena, value); + } else if (value.size() > sizeof(std::string)) { + value.shrink_to_fit(); + result.SetMedium( + arena, google::protobuf::Arena::Create(arena, std::move(value))); + } else { + char* arena_value = + reinterpret_cast(arena->AllocateAligned(value.size())); + std::memcpy(arena_value, value.data(), value.size()); + result.SetMedium(arena, absl::string_view(arena_value, value.size())); + } + return result; +} + +ByteString ByteString::Wrap(absl::string_view value, + google::protobuf::Arena* absl_nullable arena) { + ByteString result(UninitializedTag{}); + result.SetMedium(arena, value); + return result; +} + +ByteString ByteString::Wrap(const absl::Cord* absl_nonnull value, size_t offset, + size_t size, google::protobuf::Arena* absl_nullable arena) { + ByteString result(UninitializedTag{}); + result.SetLarge(arena, value, offset, size); + return result; +} + +ByteString ByteString::WrapUnsafe(absl::string_view value) { + return Wrap(value, static_cast(nullptr)); +} + +ByteString ByteString::WrapUnsafe(const absl::Cord* absl_nonnull value, + size_t offset, size_t size) { + return Wrap(value, offset, size, static_cast(nullptr)); +} + ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, google::protobuf::Arena* absl_nonnull arena) { ABSL_DCHECK(arena != nullptr); @@ -72,17 +182,18 @@ ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, rhs.GetKind() == ByteStringKind::kLarge) { // If either the left or right are absl::Cord, use absl::Cord. absl::Cord result; - result.Append(lhs.ToCord()); - result.Append(rhs.ToCord()); - return ByteString(std::move(result)); + lhs.AppendToCord(&result); + rhs.AppendToCord(&result); + return From(result, arena); } const size_t lhs_size = lhs.size(); const size_t rhs_size = rhs.size(); const size_t result_size = lhs_size + rhs_size; - ByteString result; + ByteString result(UninitializedTag{}); if (result_size <= kSmallByteStringCapacity) { // If the resulting string fits in inline storage, do it. + result.rep_.header.kind = ByteStringKind::kSmall; result.rep_.small.size = result_size; result.rep_.small.arena = arena; lhs.CopyToArray(result.rep_.small.data); @@ -93,100 +204,14 @@ ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, reinterpret_cast(arena->AllocateAligned(result_size)); lhs.CopyToArray(result_data); rhs.CopyToArray(result_data + lhs_size); + result.rep_.header.kind = ByteStringKind::kMedium; result.rep_.medium.data = result_data; result.rep_.medium.size = result_size; - result.rep_.medium.owner = - reinterpret_cast(arena) | kMetadataOwnerArenaBit; - result.rep_.header.kind = ByteStringKind::kMedium; + result.rep_.medium.arena = arena; } return result; } -ByteString::ByteString(Allocator<> allocator, absl::string_view string) { - ABSL_DCHECK_LE(string.size(), max_size()); - auto* arena = allocator.arena(); - if (string.size() <= kSmallByteStringCapacity) { - SetSmall(arena, string); - } else { - SetMedium(arena, string); - } -} - -ByteString::ByteString(Allocator<> allocator, const std::string& string) { - ABSL_DCHECK_LE(string.size(), max_size()); - auto* arena = allocator.arena(); - if (string.size() <= kSmallByteStringCapacity) { - SetSmall(arena, string); - } else { - SetMedium(arena, string); - } -} - -ByteString::ByteString(Allocator<> allocator, std::string&& string) { - ABSL_DCHECK_LE(string.size(), max_size()); - auto* arena = allocator.arena(); - if (string.size() <= kSmallByteStringCapacity) { - SetSmall(arena, string); - } else { - SetMedium(arena, std::move(string)); - } -} - -ByteString::ByteString(Allocator<> allocator, const absl::Cord& cord) { - ABSL_DCHECK_LE(cord.size(), max_size()); - auto* arena = allocator.arena(); - if (cord.size() <= kSmallByteStringCapacity) { - SetSmall(arena, cord); - } else if (arena != nullptr) { - SetMedium(arena, cord); - } else { - SetLarge(cord); - } -} - -ByteString ByteString::Borrowed(Borrower borrower, absl::string_view string) { - ABSL_DCHECK(borrower != Borrower::None()) << "Borrowing from Owner::None()"; - auto* arena = borrower.arena(); - if (string.size() <= kSmallByteStringCapacity || arena != nullptr) { - return ByteString(arena, string); - } - const auto* refcount = BorrowerRelease(borrower); - // A nullptr refcount indicates somebody called us to borrow something that - // has no owner. If this is the case, we fallback to assuming operator - // new/delete and convert it to a reference count. - if (refcount == nullptr) { - std::tie(refcount, string) = MakeReferenceCountedString(string); - } else { - StrongRef(*refcount); - } - return ByteString(refcount, string); -} - -ByteString ByteString::Borrowed(Borrower borrower, const absl::Cord& cord) { - ABSL_DCHECK(borrower != Borrower::None()) << "Borrowing from Owner::None()"; - return ByteString(borrower.arena(), cord); -} - -ByteString::ByteString(const ReferenceCount* absl_nonnull refcount, - absl::string_view string) { - ABSL_DCHECK_LE(string.size(), max_size()); - SetMedium(string, reinterpret_cast(refcount) | - kMetadataOwnerReferenceCountBit); -} - -ByteString::ByteString(ByteString::ExternalStringTag, - absl::string_view string) { - if (string.size() <= kSmallByteStringCapacity) { - SetSmall(nullptr, string); - } else { - SetExternalMedium(string); - } -} - -ByteString ByteString::FromExternal(absl::string_view string) { - return ByteString(ExternalStringTag{}, string); -} - google::protobuf::Arena* absl_nullable ByteString::GetArena() const { switch (GetKind()) { case ByteStringKind::kSmall: @@ -194,7 +219,7 @@ google::protobuf::Arena* absl_nullable ByteString::GetArena() const { case ByteStringKind::kMedium: return GetMediumArena(); case ByteStringKind::kLarge: - return nullptr; + return GetLargeArena(); } } @@ -205,7 +230,7 @@ bool ByteString::empty() const { case ByteStringKind::kMedium: return rep_.medium.size == 0; case ByteStringKind::kLarge: - return GetLarge().empty(); + return rep_.large.size == 0; } } @@ -216,18 +241,7 @@ size_t ByteString::size() const { case ByteStringKind::kMedium: return rep_.medium.size; case ByteStringKind::kLarge: - return GetLarge().size(); - } -} - -absl::string_view ByteString::Flatten() { - switch (GetKind()) { - case ByteStringKind::kSmall: - return GetSmall(); - case ByteStringKind::kMedium: - return GetMedium(); - case ByteStringKind::kLarge: - return GetLarge().Flatten(); + return rep_.large.size; } } @@ -237,8 +251,12 @@ absl::optional ByteString::TryFlat() const { return GetSmall(); case ByteStringKind::kMedium: return GetMedium(); - case ByteStringKind::kLarge: - return GetLarge().TryFlat(); + case ByteStringKind::kLarge: { + if (auto flat = rep_.large.data->TryFlat(); flat.has_value()) { + return flat->substr(rep_.large.offset, rep_.large.offset); + } + return absl::nullopt; + } } } @@ -380,22 +398,20 @@ ByteString ByteString::Substring(size_t pos, size_t npos) const { switch (GetKind()) { case ByteStringKind::kSmall: { - ByteString result; - result.rep_.header.kind = ByteStringKind::kSmall; - result.rep_.small.size = npos - pos; - std::memcpy(result.rep_.small.data, rep_.small.data + pos, - result.rep_.small.size); - result.rep_.small.arena = GetSmallArena(); + ByteString result(UninitializedTag{}); + result.SetSmall(GetSmallArena(), GetSmall().substr(pos, npos - pos)); return result; } case ByteStringKind::kMedium: { - ByteString result(*this); - result.rep_.medium.data += pos; - result.rep_.medium.size = npos - pos; + ByteString result(UninitializedTag{}); + result.SetMedium(GetMediumArena(), GetMedium().substr(pos, npos - pos)); return result; } case ByteStringKind::kLarge: - return ByteString(GetLarge().Subcord(pos, npos - pos)); + ByteString result(UninitializedTag{}); + result.SetLarge(GetLargeArena(), rep_.large.data, rep_.large.offset + pos, + npos - pos); + return result; } } @@ -407,29 +423,16 @@ void ByteString::RemovePrefix(size_t n) { switch (GetKind()) { case ByteStringKind::kSmall: std::memmove(rep_.small.data, rep_.small.data + n, rep_.small.size - n); - rep_.small.size -= n; + rep_.small.size = rep_.small.size - n; break; case ByteStringKind::kMedium: - rep_.medium.data += n; - rep_.medium.size -= n; - if (rep_.medium.size <= kSmallByteStringCapacity) { - const auto* refcount = GetMediumReferenceCount(); - SetSmall(GetMediumArena(), GetMedium()); - StrongUnref(refcount); - } + rep_.medium.data = rep_.medium.data + n; + rep_.medium.size = rep_.medium.size - n; + break; + case ByteStringKind::kLarge: + rep_.large.offset = rep_.large.offset + n; + rep_.large.size = rep_.large.size - n; break; - case ByteStringKind::kLarge: { - auto& large = GetLarge(); - const auto large_size = large.size(); - const auto new_large_pos = n; - const auto new_large_size = large_size - n; - large = large.Subcord(new_large_pos, new_large_size); - if (new_large_size <= kSmallByteStringCapacity) { - auto large_copy = std::move(large); - DestroyLarge(); - SetSmall(nullptr, large_copy); - } - } break; } } @@ -440,34 +443,19 @@ void ByteString::RemoveSuffix(size_t n) { } switch (GetKind()) { case ByteStringKind::kSmall: - rep_.small.size -= n; + rep_.small.size = rep_.small.size - n; break; case ByteStringKind::kMedium: - rep_.medium.size -= n; - if (rep_.medium.size <= kSmallByteStringCapacity) { - const auto* refcount = GetMediumReferenceCount(); - SetSmall(GetMediumArena(), GetMedium()); - StrongUnref(refcount); - } + rep_.medium.size = rep_.medium.size - n; + break; + case ByteStringKind::kLarge: + rep_.large.size = rep_.large.size - n; break; - case ByteStringKind::kLarge: { - auto& large = GetLarge(); - const auto large_size = large.size(); - const auto new_large_pos = 0; - const auto new_large_size = large_size - n; - large = large.Subcord(new_large_pos, new_large_size); - if (new_large_size <= kSmallByteStringCapacity) { - auto large_copy = std::move(large); - DestroyLarge(); - SetSmall(nullptr, large_copy); - } - } break; } } void ByteString::CopyToArray(char* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: { absl::string_view small = GetSmall(); @@ -478,8 +466,8 @@ void ByteString::CopyToArray(char* absl_nonnull out) const { std::memcpy(out, medium.data(), medium.size()); } break; case ByteStringKind::kLarge: { - const absl::Cord& large = GetLarge(); - (CopyCordToArray)(large, out); + (CopyCordToArray)(*rep_.large.data, rep_.large.offset, rep_.large.size, + out); } break; } } @@ -490,14 +478,22 @@ std::string ByteString::ToString() const { return std::string(GetSmall()); case ByteStringKind::kMedium: return std::string(GetMedium()); - case ByteStringKind::kLarge: - return static_cast(GetLarge()); + case ByteStringKind::kLarge: { + std::string result; + absl::StringResizeAndOverwrite( + result, rep_.large.size, + [this](char* buffer, size_t buffer_size) -> size_t { + (CopyCordToArray)(*rep_.large.data, rep_.large.offset, + rep_.large.size, buffer); + return rep_.large.size; + }); + return result; + } } } void ByteString::CopyToString(std::string* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: out->assign(GetSmall()); @@ -506,14 +502,19 @@ void ByteString::CopyToString(std::string* absl_nonnull out) const { out->assign(GetMedium()); break; case ByteStringKind::kLarge: - absl::CopyCordToString(GetLarge(), out); + absl::StringResizeAndOverwrite( + *out, rep_.large.size, + [this](char* buffer, size_t buffer_size) -> size_t { + (CopyCordToArray)(*rep_.large.data, rep_.large.offset, + rep_.large.size, buffer); + return rep_.large.size; + }); break; } } void ByteString::AppendToString(std::string* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: out->append(GetSmall()); @@ -522,34 +523,18 @@ void ByteString::AppendToString(std::string* absl_nonnull out) const { out->append(GetMedium()); break; case ByteStringKind::kLarge: - absl::AppendCordToString(GetLarge(), out); + (AppendCordToString)(*rep_.large.data, rep_.large.offset, rep_.large.size, + *out); break; } } -namespace { - -struct ReferenceCountReleaser { - const ReferenceCount* absl_nonnull refcount; - - void operator()() const { StrongUnref(*refcount); } -}; - -} // namespace - absl::Cord ByteString::ToCord() const& { switch (GetKind()) { case ByteStringKind::kSmall: return absl::Cord(GetSmall()); - case ByteStringKind::kMedium: { - const auto* refcount = GetMediumReferenceCount(); - if (refcount != nullptr) { - StrongRef(*refcount); - return absl::MakeCordFromExternal(GetMedium(), - ReferenceCountReleaser{refcount}); - } + case ByteStringKind::kMedium: return absl::Cord(GetMedium()); - } case ByteStringKind::kLarge: return GetLarge(); } @@ -559,16 +544,8 @@ absl::Cord ByteString::ToCord() && { switch (GetKind()) { case ByteStringKind::kSmall: return absl::Cord(GetSmall()); - case ByteStringKind::kMedium: { - const auto* refcount = GetMediumReferenceCount(); - if (refcount != nullptr) { - auto medium = GetMedium(); - SetSmallEmpty(nullptr); - return absl::MakeCordFromExternal(medium, - ReferenceCountReleaser{refcount}); - } + case ByteStringKind::kMedium: return absl::Cord(GetMedium()); - } case ByteStringKind::kLarge: return GetLarge(); } @@ -576,21 +553,13 @@ absl::Cord ByteString::ToCord() && { void ByteString::CopyToCord(absl::Cord* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: *out = absl::Cord(GetSmall()); break; - case ByteStringKind::kMedium: { - const auto* refcount = GetMediumReferenceCount(); - if (refcount != nullptr) { - StrongRef(*refcount); - *out = absl::MakeCordFromExternal(GetMedium(), - ReferenceCountReleaser{refcount}); - } else { - *out = absl::Cord(GetMedium()); - } - } break; + case ByteStringKind::kMedium: + *out = absl::Cord(GetMedium()); + break; case ByteStringKind::kLarge: *out = GetLarge(); break; @@ -599,21 +568,13 @@ void ByteString::CopyToCord(absl::Cord* absl_nonnull out) const { void ByteString::AppendToCord(absl::Cord* absl_nonnull out) const { ABSL_DCHECK(out != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: out->Append(GetSmall()); break; - case ByteStringKind::kMedium: { - const auto* refcount = GetMediumReferenceCount(); - if (refcount != nullptr) { - StrongRef(*refcount); - out->Append(absl::MakeCordFromExternal( - GetMedium(), ReferenceCountReleaser{refcount})); - } else { - out->Append(GetMedium()); - } - } break; + case ByteStringKind::kMedium: + out->Append(GetMedium()); + break; case ByteStringKind::kLarge: out->Append(GetLarge()); break; @@ -623,17 +584,22 @@ void ByteString::AppendToCord(absl::Cord* absl_nonnull out) const { absl::string_view ByteString::ToStringView( std::string* absl_nonnull scratch) const { ABSL_DCHECK(scratch != nullptr); - switch (GetKind()) { case ByteStringKind::kSmall: return GetSmall(); case ByteStringKind::kMedium: return GetMedium(); case ByteStringKind::kLarge: - if (auto flat = GetLarge().TryFlat(); flat) { - return *flat; + if (auto flat = rep_.large.data->TryFlat(); flat.has_value()) { + return flat->substr(rep_.large.offset, rep_.large.size); } - absl::CopyCordToString(GetLarge(), scratch); + absl::StringResizeAndOverwrite( + *scratch, rep_.large.size, + [this](char* buffer, size_t buffer_size) -> size_t { + (CopyCordToArray)(*rep_.large.data, rep_.large.offset, + rep_.large.size, buffer); + return rep_.large.size; + }); return absl::string_view(*scratch); } } @@ -652,203 +618,28 @@ absl::string_view ByteString::AsStringView() const { } } -google::protobuf::Arena* absl_nullable ByteString::GetMediumArena( - const MediumByteStringRep& rep) { - if ((rep.owner & kMetadataOwnerBits) == kMetadataOwnerArenaBit) { - return reinterpret_cast(rep.owner & - kMetadataOwnerPointerMask); - } - return nullptr; -} - -const ReferenceCount* absl_nullable ByteString::GetMediumReferenceCount( - const MediumByteStringRep& rep) { - if ((rep.owner & kMetadataOwnerBits) == kMetadataOwnerReferenceCountBit) { - return reinterpret_cast(rep.owner & - kMetadataOwnerPointerMask); - } - return nullptr; -} - -void ByteString::Construct(const ByteString& other, - absl::optional> allocator) { - switch (other.GetKind()) { - case ByteStringKind::kSmall: - rep_.small = other.rep_.small; - if (allocator.has_value()) { - rep_.small.arena = allocator->arena(); - } - break; - case ByteStringKind::kMedium: - if (allocator.has_value() && - allocator->arena() != other.GetMediumArena()) { - SetMedium(allocator->arena(), other.GetMedium()); - } else { - rep_.medium = other.rep_.medium; - StrongRef(GetMediumReferenceCount()); - } - break; - case ByteStringKind::kLarge: - if (allocator.has_value() && allocator->arena() != nullptr) { - SetMedium(allocator->arena(), other.GetLarge()); - } else { - SetLarge(other.GetLarge()); - } - break; - } -} - -void ByteString::Construct(ByteString& other, - absl::optional> allocator) { - switch (other.GetKind()) { - case ByteStringKind::kSmall: - rep_.small = other.rep_.small; - if (allocator.has_value()) { - rep_.small.arena = allocator->arena(); - } - break; - case ByteStringKind::kMedium: - if (allocator.has_value() && - allocator->arena() != other.GetMediumArena()) { - SetMedium(allocator->arena(), other.GetMedium()); - } else { - rep_.medium = other.rep_.medium; - other.rep_.medium.owner = 0; - } - break; - case ByteStringKind::kLarge: - if (allocator.has_value() && allocator->arena() != nullptr) { - SetMedium(allocator->arena(), other.GetLarge()); - } else { - SetLarge(std::move(other.GetLarge())); - } - break; - } -} - -void ByteString::CopyFrom(const ByteString& other) { - ABSL_DCHECK_NE(&other, this); - - switch (other.GetKind()) { - case ByteStringKind::kSmall: - switch (GetKind()) { - case ByteStringKind::kSmall: - break; - case ByteStringKind::kMedium: - DestroyMedium(); - break; - case ByteStringKind::kLarge: - DestroyLarge(); - break; - } - rep_.small = other.rep_.small; - break; - case ByteStringKind::kMedium: - switch (GetKind()) { - case ByteStringKind::kSmall: - rep_.medium = other.rep_.medium; - StrongRef(GetMediumReferenceCount()); - break; - case ByteStringKind::kMedium: - StrongRef(other.GetMediumReferenceCount()); - DestroyMedium(); - rep_.medium = other.rep_.medium; - break; - case ByteStringKind::kLarge: - DestroyLarge(); - rep_.medium = other.rep_.medium; - StrongRef(GetMediumReferenceCount()); - break; - } - break; - case ByteStringKind::kLarge: - switch (GetKind()) { - case ByteStringKind::kSmall: - SetLarge(other.GetLarge()); - break; - case ByteStringKind::kMedium: - DestroyMedium(); - SetLarge(other.GetLarge()); - break; - case ByteStringKind::kLarge: - GetLarge() = other.GetLarge(); - break; - } - break; - } -} - -void ByteString::MoveFrom(ByteString& other) { - ABSL_DCHECK_NE(&other, this); - - switch (other.GetKind()) { - case ByteStringKind::kSmall: - switch (GetKind()) { - case ByteStringKind::kSmall: - break; - case ByteStringKind::kMedium: - DestroyMedium(); - break; - case ByteStringKind::kLarge: - DestroyLarge(); - break; - } - rep_.small = other.rep_.small; - break; - case ByteStringKind::kMedium: - switch (GetKind()) { - case ByteStringKind::kSmall: - rep_.medium = other.rep_.medium; - break; - case ByteStringKind::kMedium: - DestroyMedium(); - rep_.medium = other.rep_.medium; - break; - case ByteStringKind::kLarge: - DestroyLarge(); - rep_.medium = other.rep_.medium; - break; - } - other.rep_.medium.owner = 0; - break; - case ByteStringKind::kLarge: - switch (GetKind()) { - case ByteStringKind::kSmall: - SetLarge(std::move(other.GetLarge())); - break; - case ByteStringKind::kMedium: - DestroyMedium(); - SetLarge(std::move(other.GetLarge())); - break; - case ByteStringKind::kLarge: - GetLarge() = std::move(other.GetLarge()); - break; - } - break; - } -} - ByteString ByteString::Clone(google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); - switch (GetKind()) { - case ByteStringKind::kSmall: - return ByteString(arena, GetSmall()); + case ByteStringKind::kSmall: { + ByteString result(UninitializedTag{}); + result.SetSmall(arena, GetSmall()); + return result; + } case ByteStringKind::kMedium: { google::protobuf::Arena* absl_nullable other_arena = GetMediumArena(); - if (arena != nullptr) { - if (arena == other_arena) { - return *this; - } - return ByteString(arena, GetMedium()); + if (other_arena != arena) { + return From(GetMedium(), arena); } - if (other_arena != nullptr) { - return ByteString(arena, GetMedium()); + return *this; + } + case ByteStringKind::kLarge: { + google::protobuf::Arena* absl_nullable other_arena = GetLargeArena(); + if (other_arena != arena) { + return From(GetLarge(), arena); } return *this; } - case ByteStringKind::kLarge: - return ByteString(arena, GetLarge()); } } @@ -866,80 +657,6 @@ void ByteString::HashValue(absl::HashState state) const { } } -void ByteString::Swap(ByteString& other) { - ABSL_DCHECK_NE(&other, this); - using std::swap; - - switch (other.GetKind()) { - case ByteStringKind::kSmall: - switch (GetKind()) { - case ByteStringKind::kSmall: - // small <=> small - swap(rep_.small, other.rep_.small); - break; - case ByteStringKind::kMedium: - // medium <=> small - swap(rep_, other.rep_); - break; - case ByteStringKind::kLarge: { - absl::Cord cord = std::move(GetLarge()); - DestroyLarge(); - rep_ = other.rep_; - other.SetLarge(std::move(cord)); - } break; - } - break; - case ByteStringKind::kMedium: - switch (GetKind()) { - case ByteStringKind::kSmall: - swap(rep_, other.rep_); - break; - case ByteStringKind::kMedium: - swap(rep_.medium, other.rep_.medium); - break; - case ByteStringKind::kLarge: { - absl::Cord cord = std::move(GetLarge()); - DestroyLarge(); - rep_ = other.rep_; - other.SetLarge(std::move(cord)); - } break; - } - break; - case ByteStringKind::kLarge: - switch (GetKind()) { - case ByteStringKind::kSmall: { - absl::Cord cord = std::move(other.GetLarge()); - other.DestroyLarge(); - other.rep_.small = rep_.small; - SetLarge(std::move(cord)); - } break; - case ByteStringKind::kMedium: { - absl::Cord cord = std::move(other.GetLarge()); - other.DestroyLarge(); - other.rep_.medium = rep_.medium; - SetLarge(std::move(cord)); - } break; - case ByteStringKind::kLarge: - swap(GetLarge(), other.GetLarge()); - break; - } - break; - } -} - -void ByteString::Destroy() { - switch (GetKind()) { - case ByteStringKind::kSmall: - break; - case ByteStringKind::kMedium: - DestroyMedium(); - break; - case ByteStringKind::kLarge: - DestroyLarge(); - break; - } -} - void ByteString::SetSmall(google::protobuf::Arena* absl_nullable arena, absl::string_view string) { ABSL_DCHECK_LE(string.size(), kSmallByteStringCapacity); @@ -962,82 +679,27 @@ void ByteString::SetSmall(google::protobuf::Arena* absl_nullable arena, void ByteString::SetMedium(google::protobuf::Arena* absl_nullable arena, absl::string_view string) { - ABSL_DCHECK_GT(string.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = string.size(); - if (arena != nullptr) { - char* data = static_cast( - arena->AllocateAligned(rep_.medium.size, alignof(char))); - std::memcpy(data, string.data(), rep_.medium.size); - rep_.medium.data = data; - rep_.medium.owner = - reinterpret_cast(arena) | kMetadataOwnerArenaBit; - } else { - auto pair = MakeReferenceCountedString(string); - rep_.medium.data = pair.second.data(); - rep_.medium.owner = reinterpret_cast(pair.first) | - kMetadataOwnerReferenceCountBit; - } -} - -void ByteString::SetExternalMedium(absl::string_view string) { - ABSL_DCHECK_GT(string.size(), kSmallByteStringCapacity); rep_.header.kind = ByteStringKind::kMedium; rep_.medium.size = string.size(); rep_.medium.data = string.data(); - rep_.medium.owner = 0; -} - -void ByteString::SetMedium(google::protobuf::Arena* absl_nullable arena, - std::string&& string) { - ABSL_DCHECK_GT(string.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = string.size(); - if (arena != nullptr) { - auto* data = google::protobuf::Arena::Create(arena, std::move(string)); - rep_.medium.data = data->data(); - rep_.medium.owner = - reinterpret_cast(arena) | kMetadataOwnerArenaBit; - } else { - auto pair = MakeReferenceCountedString(std::move(string)); - rep_.medium.data = pair.second.data(); - rep_.medium.owner = reinterpret_cast(pair.first) | - kMetadataOwnerReferenceCountBit; - } -} - -void ByteString::SetMedium(google::protobuf::Arena* absl_nonnull arena, - const absl::Cord& cord) { - ABSL_DCHECK_GT(cord.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = cord.size(); - char* data = static_cast( - arena->AllocateAligned(rep_.medium.size, alignof(char))); - (CopyCordToArray)(cord, data); - rep_.medium.data = data; - rep_.medium.owner = - reinterpret_cast(arena) | kMetadataOwnerArenaBit; -} - -void ByteString::SetMedium(absl::string_view string, uintptr_t owner) { - ABSL_DCHECK_GT(string.size(), kSmallByteStringCapacity); - ABSL_DCHECK_NE(owner, 0); - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = string.size(); - rep_.medium.data = string.data(); - rep_.medium.owner = owner; -} - -void ByteString::SetLarge(const absl::Cord& cord) { - ABSL_DCHECK_GT(cord.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kLarge; - ::new (static_cast(&rep_.large.data[0])) absl::Cord(cord); + rep_.medium.arena = arena; } -void ByteString::SetLarge(absl::Cord&& cord) { - ABSL_DCHECK_GT(cord.size(), kSmallByteStringCapacity); +void ByteString::SetLarge(google::protobuf::Arena* absl_nullable arena, + const absl::Cord* absl_nonnull cord, size_t offset, + size_t size) { + ABSL_DCHECK_LE(offset, cord->size()); + ABSL_DCHECK_LE(offset, kLargeByteStringMaxSize); rep_.header.kind = ByteStringKind::kLarge; - ::new (static_cast(&rep_.large.data[0])) absl::Cord(std::move(cord)); + rep_.large.offset = offset; + if (size == static_cast(-1)) { + size = cord->size() - offset; + } + ABSL_DCHECK_LE(size, cord->size() - offset); + ABSL_DCHECK_LE(size, kLargeByteStringMaxSize); + rep_.large.size = size; + rep_.large.data = cord; + rep_.large.arena = arena; } absl::string_view LegacyByteString(const ByteString& string, bool stable, diff --git a/common/internal/byte_string.h b/common/internal/byte_string.h index c576e5634..de11f8a9e 100644 --- a/common/internal/byte_string.h +++ b/common/internal/byte_string.h @@ -17,25 +17,19 @@ #include #include -#include +#include #include #include -#include #include #include "absl/base/attributes.h" #include "absl/base/nullability.h" -#include "absl/base/optimization.h" #include "absl/functional/overload.h" #include "absl/hash/hash.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/arena.h" -#include "common/internal/reference_count.h" -#include "common/memory.h" #include "google/protobuf/arena.h" namespace cel { @@ -46,15 +40,7 @@ class StringValue; namespace common_internal { -// absl::Cord is trivially relocatable IFF we are not using ASan or MSan. When -// using ASan or MSan absl::Cord will poison/unpoison its inline storage. -#if defined(ABSL_HAVE_ADDRESS_SANITIZER) || defined(ABSL_HAVE_MEMORY_SANITIZER) -#define CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI -#else -#define CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI ABSL_ATTRIBUTE_TRIVIAL_ABI -#endif - -class CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI [[nodiscard]] ByteString; +class [[nodiscard]] ByteString; struct ByteStringTestFriend; @@ -76,11 +62,11 @@ inline std::ostream& operator<<(std::ostream& out, ByteStringKind kind) { } // Representation of small strings in ByteString, which are stored in place. -struct CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI SmallByteStringRep final { +struct SmallByteStringRep final { #ifdef _MSC_VER #pragma pack(push, 1) #endif - struct ABSL_ATTRIBUTE_PACKED CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI { + struct ABSL_ATTRIBUTE_PACKED { std::uint8_t kind : 2; std::uint8_t size : 6; }; @@ -98,6 +84,9 @@ inline constexpr size_t kMediumByteStringSizeBits = sizeof(size_t) * 8 - 2; inline constexpr size_t kMediumByteStringMaxSize = (size_t{1} << kMediumByteStringSizeBits) - 1; +inline constexpr size_t kLargeByteStringMaxSize = + (size_t{1} << kMediumByteStringSizeBits) - 1; + inline constexpr size_t kByteStringViewSizeBits = sizeof(size_t) * 8 - 1; inline constexpr size_t kByteStringViewMaxSize = (size_t{1} << kByteStringViewSizeBits) - 1; @@ -105,43 +94,45 @@ inline constexpr size_t kByteStringViewMaxSize = // Representation of medium strings in ByteString. These are either owned by an // arena or managed by a reference count. This is encoded in `owner` following // the same semantics as `cel::Owner`. -struct CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI MediumByteStringRep final { +struct MediumByteStringRep final { #ifdef _MSC_VER #pragma pack(push, 1) #endif - struct ABSL_ATTRIBUTE_PACKED CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI { + struct ABSL_ATTRIBUTE_PACKED { size_t kind : 2; size_t size : kMediumByteStringSizeBits; }; #ifdef _MSC_VER #pragma pack(pop) #endif - const char* data; - uintptr_t owner; + const char* absl_nullability_unknown data; + google::protobuf::Arena* absl_nullable arena; }; // Representation of large strings in ByteString. These are stored as -// `absl::Cord` and never owned by an arena. -struct CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI LargeByteStringRep final { +// a pointer to `absl::Cord`. +struct LargeByteStringRep final { #ifdef _MSC_VER #pragma pack(push, 1) #endif - struct ABSL_ATTRIBUTE_PACKED CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI { + struct ABSL_ATTRIBUTE_PACKED { size_t kind : 2; - size_t padding : kMediumByteStringSizeBits; + size_t offset : kMediumByteStringSizeBits / 2; + size_t size : kMediumByteStringSizeBits / 2; }; #ifdef _MSC_VER #pragma pack(pop) #endif - alignas(absl::Cord) std::byte data[sizeof(absl::Cord)]; + const absl::Cord* absl_nullability_unknown data; + google::protobuf::Arena* absl_nullable arena; }; // Representation of ByteString. -union CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI ByteStringRep final { +union ByteStringRep final { #ifdef _MSC_VER #pragma pack(push, 1) #endif - struct ABSL_ATTRIBUTE_PACKED CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI { + struct ABSL_ATTRIBUTE_PACKED { ByteStringKind kind : 2; } header; #ifdef _MSC_VER @@ -166,91 +157,56 @@ absl::string_view LegacyByteString(const ByteString& string, bool stable, // string is constructed the allocator will not and cannot change. Copying and // moving between different allocators is supported and dealt with // transparently by copying. -class CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI [[nodiscard]] -ByteString final { +class [[nodiscard]] ByteString final { public: - static ByteString Concat(const ByteString& lhs, const ByteString& rhs, - google::protobuf::Arena* absl_nonnull arena); - - ByteString() : ByteString(NewDeleteAllocator()) {} - - explicit ByteString(const char* absl_nullable string) - : ByteString(NewDeleteAllocator(), string) {} - - explicit ByteString(absl::string_view string) - : ByteString(NewDeleteAllocator(), string) {} - - explicit ByteString(const std::string& string) - : ByteString(NewDeleteAllocator(), string) {} - - explicit ByteString(std::string&& string) - : ByteString(NewDeleteAllocator(), std::move(string)) {} - - explicit ByteString(const absl::Cord& cord) - : ByteString(NewDeleteAllocator(), cord) {} - - ByteString(const ByteString& other) noexcept { - Construct(other, /*allocator=*/absl::nullopt); - } - - ByteString(ByteString&& other) noexcept { - Construct(other, /*allocator=*/absl::nullopt); - } - - explicit ByteString(Allocator<> allocator) { - SetSmallEmpty(allocator.arena()); + static ByteString From(const char* absl_nullable value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); + static ByteString From(absl::string_view value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); + static ByteString From(const absl::Cord& value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); + static ByteString From(std::string&& value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); + + static ByteString Wrap(absl::string_view value, + google::protobuf::Arena* absl_nullable arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); + static ByteString Wrap( + const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return Wrap(value, 0, value->size(), arena); } - - ByteString(Allocator<> allocator, const char* absl_nullable string) - : ByteString(allocator, absl::NullSafeStringView(string)) {} - - ByteString(Allocator<> allocator, absl::string_view string); - - ByteString(Allocator<> allocator, const std::string& string); - - ByteString(Allocator<> allocator, std::string&& string); - - ByteString(Allocator<> allocator, const absl::Cord& cord); - - ByteString(Allocator<> allocator, const ByteString& other) { - Construct(other, allocator); - } - - ByteString(Allocator<> allocator, ByteString&& other) { - Construct(other, allocator); + static ByteString Wrap( + const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + size_t offset, size_t size, + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND); + static ByteString Wrap(std::nullptr_t, google::protobuf::Arena*) = delete; + static ByteString Wrap(std::nullptr_t, size_t, size_t, + google::protobuf::Arena*) = delete; + static ByteString Wrap(std::string&& value, google::protobuf::Arena*) = delete; + + static ByteString WrapUnsafe(absl::string_view value); + static ByteString WrapUnsafe(const absl::Cord* absl_nonnull value) { + return WrapUnsafe(value, 0, value->size()); } + static ByteString WrapUnsafe(const absl::Cord* absl_nonnull value, + size_t offset, size_t size); + static ByteString WrapUnsafe(std::nullptr_t) = delete; + static ByteString WrapUnsafe(std::nullptr_t, size_t, size_t) = delete; - ByteString(Borrower borrower, - const char* absl_nullable string ABSL_ATTRIBUTE_LIFETIME_BOUND) - : ByteString(borrower, absl::NullSafeStringView(string)) {} - - ByteString(Borrower borrower, - absl::string_view string ABSL_ATTRIBUTE_LIFETIME_BOUND) - : ByteString(Borrowed(borrower, string)) {} - - ByteString(Borrower borrower, - const absl::Cord& cord ABSL_ATTRIBUTE_LIFETIME_BOUND) - : ByteString(Borrowed(borrower, cord)) {} - - // Creates a medium byte string that is backed by an external string. Should - // only be called from explicit 'Unsafe' factories. - static ByteString FromExternal(absl::string_view string); + static ByteString Concat(const ByteString& lhs, const ByteString& rhs, + google::protobuf::Arena* absl_nonnull arena); - ~ByteString() { Destroy(); } + ByteString() noexcept { SetSmallEmpty(nullptr); } - ByteString& operator=(const ByteString& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - CopyFrom(other); - } - return *this; - } - - ByteString& operator=(ByteString&& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - MoveFrom(other); - } - return *this; - } + ByteString(const ByteString&) = default; + ByteString(ByteString&&) = default; + ByteString& operator=(const ByteString&) = default; + ByteString& operator=(ByteString&&) = default; bool empty() const; @@ -258,8 +214,6 @@ ByteString final { size_t max_size() const { return kByteStringViewMaxSize; } - absl::string_view Flatten() ABSL_ATTRIBUTE_LIFETIME_BOUND; - absl::optional TryFlat() const ABSL_ATTRIBUTE_LIFETIME_BOUND; @@ -339,10 +293,9 @@ ByteString final { } } - friend void swap(ByteString& lhs, ByteString& rhs) { - if (&lhs != &rhs) { - lhs.Swap(rhs); - } + friend void swap(ByteString& lhs, ByteString& rhs) noexcept { + using std::swap; + swap(lhs.rep_, rhs.rep_); } template @@ -360,21 +313,12 @@ ByteString final { friend absl::string_view LegacyByteString(const ByteString& string, bool stable, google::protobuf::Arena* absl_nonnull arena); - friend struct cel::ArenaTraits; - - struct ExternalStringTag {}; - - static ByteString Borrowed(Borrower borrower, - absl::string_view string - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static ByteString Borrowed( - Borrower borrower, const absl::Cord& cord ABSL_ATTRIBUTE_LIFETIME_BOUND); - - ByteString(const ReferenceCount* absl_nonnull refcount, - absl::string_view string); + struct UninitializedTag { + explicit UninitializedTag() = default; + }; - ByteString(ExternalStringTag, absl::string_view string); + explicit ByteString(UninitializedTag) {} constexpr ByteStringKind GetKind() const { return rep_.header.kind; } @@ -412,39 +356,28 @@ ByteString final { } static google::protobuf::Arena* absl_nullable GetMediumArena( - const MediumByteStringRep& rep); - - const ReferenceCount* absl_nullable GetMediumReferenceCount() const { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kMedium); - return GetMediumReferenceCount(rep_.medium); + const MediumByteStringRep& rep) { + return rep.arena; } - static const ReferenceCount* absl_nullable GetMediumReferenceCount( - const MediumByteStringRep& rep); - - uintptr_t GetMediumOwner() const { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kMedium); - return rep_.medium.owner; + static absl::Cord GetLarge( + const LargeByteStringRep& rep ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return rep.data->Subcord(rep.offset, rep.size); } - absl::Cord& GetLarge() ABSL_ATTRIBUTE_LIFETIME_BOUND { + absl::Cord GetLarge() const ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kLarge); return GetLarge(rep_.large); } - static absl::Cord& GetLarge( - LargeByteStringRep& rep ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return *std::launder(reinterpret_cast(&rep.data[0])); - } - - const absl::Cord& GetLarge() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + google::protobuf::Arena* absl_nullable GetLargeArena() const { ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kLarge); - return GetLarge(rep_.large); + return GetLargeArena(rep_.large); } - static const absl::Cord& GetLarge( + static google::protobuf::Arena* absl_nullable GetLargeArena( const LargeByteStringRep& rep ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return *std::launder(reinterpret_cast(&rep.data[0])); + return rep.arena; } void SetSmallEmpty(google::protobuf::Arena* absl_nullable arena) { @@ -459,48 +392,14 @@ ByteString final { void SetMedium(google::protobuf::Arena* absl_nullable arena, absl::string_view string); - // This is used to create a medium byte string that is backed by an external - // string. Should only be called from explicit 'Unsafe' factories. - void SetExternalMedium(absl::string_view string); - - void SetMedium(google::protobuf::Arena* absl_nullable arena, std::string&& string); - - void SetMedium(google::protobuf::Arena* absl_nonnull arena, const absl::Cord& cord); - - void SetMedium(absl::string_view string, uintptr_t owner); - - void SetLarge(const absl::Cord& cord); - - void SetLarge(absl::Cord&& cord); - - void Swap(ByteString& other); - - void Construct(const ByteString& other, - absl::optional> allocator); - - void Construct(ByteString& other, absl::optional> allocator); - - void CopyFrom(const ByteString& other); - - void MoveFrom(ByteString& other); - - void Destroy(); - - void DestroyMedium() { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kMedium); - DestroyMedium(rep_.medium); + void SetMedium(google::protobuf::Arena* absl_nullable arena, + const std::string* absl_nonnull string) { + SetMedium(arena, absl::string_view(*string)); } - static void DestroyMedium(const MediumByteStringRep& rep) { - StrongUnref(GetMediumReferenceCount(rep)); - } - - void DestroyLarge() { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kLarge); - DestroyLarge(rep_.large); - } - - static void DestroyLarge(LargeByteStringRep& rep) { GetLarge(rep).~Cord(); } + void SetLarge(google::protobuf::Arena* absl_nullable arena, + const absl::Cord* absl_nonnull cord, size_t offset = 0, + size_t size = static_cast(-1)); void CopyToArray(char* absl_nonnull out) const; @@ -662,27 +561,8 @@ inline bool operator>=(const absl::Cord& lhs, const ByteString& rhs) { return -rhs.Compare(lhs) >= 0; } -#undef CEL_COMMON_INTERNAL_BYTE_STRING_TRIVIAL_ABI - } // namespace common_internal -template <> -struct ArenaTraits { - using constructible = std::true_type; - - static bool trivially_destructible( - const common_internal::ByteString& byte_string) { - switch (byte_string.GetKind()) { - case common_internal::ByteStringKind::kSmall: - return true; - case common_internal::ByteStringKind::kMedium: - return byte_string.GetMediumReferenceCount() == nullptr; - case common_internal::ByteStringKind::kLarge: - return false; - } - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_BYTE_STRING_H_ diff --git a/common/internal/byte_string_test.cc b/common/internal/byte_string_test.cc index c4ae0eae6..b2353ac2d 100644 --- a/common/internal/byte_string_test.cc +++ b/common/internal/byte_string_test.cc @@ -24,10 +24,6 @@ #include "absl/strings/cord.h" #include "absl/strings/cord_test_helpers.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/internal/reference_count.h" -#include "common/memory.h" #include "internal/testing.h" #include "google/protobuf/arena.h" @@ -67,17 +63,9 @@ TEST(ByteStringKind, Ostream) { } } -class ByteStringTest : public TestWithParam, - public ByteStringTestFriend { +class ByteStringTest : public ByteStringTestFriend, public ::testing::Test { public: - Allocator<> GetAllocator() { - switch (GetParam()) { - case AllocatorKind::kNewDelete: - return NewDeleteAllocator<>{}; - case AllocatorKind::kArena: - return ArenaAllocator<>(&arena_); - } - } + google::protobuf::Arena* GetArena() { return &arena_; } private: google::protobuf::Arena arena_; @@ -117,686 +105,165 @@ const absl::Cord& GetMediumOrLargeFragmentedCord() { return *medium_or_large; } -TEST_P(ByteStringTest, Default) { - ByteString byte_string = ByteString(GetAllocator(), ""); +TEST_F(ByteStringTest, Default) { + ByteString byte_string = ByteString(); EXPECT_THAT(byte_string, SizeIs(0)); EXPECT_THAT(byte_string, IsEmpty()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); } -TEST_P(ByteStringTest, ConstructNullDataStringView) { - ByteString byte_string(GetAllocator(), absl::string_view()); +TEST_F(ByteStringTest, ConstructNullDataStringView) { + ByteString byte_string = ByteString::From(absl::string_view(), GetArena()); EXPECT_THAT(byte_string, IsEmpty()); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructSmallCString) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallString().c_str()); +TEST_F(ByteStringTest, ConstructSmallCString) { + ByteString byte_string = + ByteString::From(GetSmallString().c_str(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructMediumCString) { +TEST_F(ByteStringTest, ConstructMediumCString) { ByteString byte_string = - ByteString(GetAllocator(), GetMediumString().c_str()); + ByteString::From(GetMediumString().c_str(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetMediumStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructSmallRValueString) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallString()); +TEST_F(ByteStringTest, ConstructSmallRValueString) { + ByteString byte_string = ByteString::From(GetSmallString(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructSmallLValueString) { - ByteString byte_string = ByteString( - GetAllocator(), static_cast(GetSmallString())); +TEST_F(ByteStringTest, ConstructSmallLValueString) { + ByteString byte_string = ByteString::From( + static_cast(GetSmallString()), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructMediumRValueString) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumString()); +TEST_F(ByteStringTest, ConstructMediumRValueString) { + ByteString byte_string = ByteString::From(GetMediumString(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetMediumStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructMediumLValueString) { - ByteString byte_string = ByteString( - GetAllocator(), static_cast(GetMediumString())); +TEST_F(ByteStringTest, ConstructMediumLValueString) { + ByteString byte_string = ByteString::From( + static_cast(GetMediumString()), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetMediumStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructSmallCord) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallCord()); +TEST_F(ByteStringTest, ConstructSmallCord) { + ByteString byte_string = ByteString::From(GetSmallCord(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetSmallStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST_P(ByteStringTest, ConstructMediumOrLargeCord) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, ConstructMediumOrLargeCord) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_THAT(byte_string, SizeIs(GetMediumStringView().size())); EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetMediumStringView()); - if (GetAllocator().arena() == nullptr) { - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); - } else { - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - } - EXPECT_EQ(byte_string.GetArena(), GetAllocator().arena()); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); + EXPECT_EQ(byte_string.GetArena(), GetArena()); } -TEST(ByteStringTest, BorrowedUnownedString) { -#ifdef NDEBUG - ByteString byte_string = ByteString(Owner::None(), GetMediumStringView()); +TEST_F(ByteStringTest, BorrowedArenaSmallString) { + ByteString byte_string = ByteString::Wrap(GetSmallStringView(), GetArena()); EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), nullptr); - EXPECT_EQ(byte_string, GetMediumStringView()); -#else - EXPECT_DEBUG_DEATH( - static_cast(ByteString(Owner::None(), GetMediumStringView())), - ::testing::_); -#endif -} - -TEST(ByteStringTest, BorrowedUnownedCord) { -#ifdef NDEBUG - ByteString byte_string = ByteString(Owner::None(), GetMediumOrLargeCord()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kLarge); - EXPECT_EQ(byte_string.GetArena(), nullptr); - EXPECT_EQ(byte_string, GetMediumOrLargeCord()); -#else - EXPECT_DEBUG_DEATH( - static_cast(ByteString(Owner::None(), GetMediumOrLargeCord())), - ::testing::_); -#endif -} - -TEST(ByteStringTest, BorrowedReferenceCountSmallString) { - auto* refcount = new ReferenceCounted(); - Owner owner = Owner::ReferenceCount(refcount); - StrongUnref(refcount); - ByteString byte_string = ByteString(owner, GetSmallStringView()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), nullptr); + EXPECT_EQ(byte_string.GetArena(), GetArena()); EXPECT_EQ(byte_string, GetSmallStringView()); } -TEST(ByteStringTest, BorrowedReferenceCountMediumString) { - auto* refcount = new ReferenceCounted(); - Owner owner = Owner::ReferenceCount(refcount); - StrongUnref(refcount); - ByteString byte_string = ByteString(owner, GetMediumStringView()); +TEST_F(ByteStringTest, BorrowedArenaMediumString) { + ByteString byte_string = ByteString::Wrap(GetMediumStringView(), GetArena()); EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), nullptr); + EXPECT_EQ(byte_string.GetArena(), GetArena()); EXPECT_EQ(byte_string, GetMediumStringView()); } -TEST(ByteStringTest, BorrowedArenaSmallString) { - google::protobuf::Arena arena; - ByteString byte_string = - ByteString(Owner::Arena(&arena), GetSmallStringView()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), &arena); - EXPECT_EQ(byte_string, GetSmallStringView()); -} - -TEST(ByteStringTest, BorrowedArenaMediumString) { - google::protobuf::Arena arena; +TEST_F(ByteStringTest, BorrowedArenaCord) { ByteString byte_string = - ByteString(Owner::Arena(&arena), GetMediumStringView()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), - ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), &arena); - EXPECT_EQ(byte_string, GetMediumStringView()); -} - -TEST(ByteStringTest, BorrowedReferenceCountCord) { - auto* refcount = new ReferenceCounted(); - Owner owner = Owner::ReferenceCount(refcount); - StrongUnref(refcount); - ByteString byte_string = ByteString(owner, GetMediumOrLargeCord()); + ByteString::Wrap(&GetMediumOrLargeCord(), GetArena()); EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), ByteStringKind::kLarge); - EXPECT_EQ(byte_string.GetArena(), nullptr); - EXPECT_EQ(byte_string, GetMediumOrLargeCord()); -} - -TEST(ByteStringTest, BorrowedArenaCord) { - google::protobuf::Arena arena; - Owner owner = Owner::Arena(&arena); - ByteString byte_string = ByteString(owner, GetMediumOrLargeCord()); - EXPECT_EQ(ByteStringTestFriend::GetKind(byte_string), - ByteStringKind::kMedium); - EXPECT_EQ(byte_string.GetArena(), &arena); + EXPECT_EQ(byte_string.GetArena(), GetArena()); EXPECT_EQ(byte_string, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, CopyConstruct) { - ByteString small_byte_string = - ByteString(GetAllocator(), GetSmallStringView()); - ByteString medium_byte_string = - ByteString(GetAllocator(), GetMediumStringView()); - ByteString large_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); - - EXPECT_EQ(ByteString(NewDeleteAllocator(), small_byte_string), - small_byte_string); - EXPECT_EQ(ByteString(NewDeleteAllocator(), medium_byte_string), - medium_byte_string); - EXPECT_EQ(ByteString(NewDeleteAllocator(), large_byte_string), - large_byte_string); - - google::protobuf::Arena arena; - EXPECT_EQ(ByteString(ArenaAllocator(&arena), small_byte_string), - small_byte_string); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), medium_byte_string), - medium_byte_string); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), large_byte_string), - large_byte_string); - - EXPECT_EQ(ByteString(GetAllocator(), small_byte_string), small_byte_string); - EXPECT_EQ(ByteString(GetAllocator(), medium_byte_string), medium_byte_string); - EXPECT_EQ(ByteString(GetAllocator(), large_byte_string), large_byte_string); - - EXPECT_EQ(ByteString(small_byte_string), small_byte_string); - EXPECT_EQ(ByteString(medium_byte_string), medium_byte_string); - EXPECT_EQ(ByteString(large_byte_string), large_byte_string); -} - -TEST_P(ByteStringTest, CopyConstructFromExternal) { - ByteString small_byte_string = ByteString::FromExternal(GetSmallStringView()); - ByteString medium_byte_string = - ByteString::FromExternal(GetMediumStringView()); - - EXPECT_EQ(ByteString(NewDeleteAllocator(), small_byte_string), - small_byte_string); - EXPECT_EQ(ByteString(NewDeleteAllocator(), medium_byte_string), - medium_byte_string); - - google::protobuf::Arena arena; - EXPECT_EQ(ByteString(ArenaAllocator(&arena), small_byte_string), - small_byte_string); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), medium_byte_string), - medium_byte_string); - - EXPECT_EQ(ByteString(GetAllocator(), small_byte_string), small_byte_string); - EXPECT_EQ(ByteString(GetAllocator(), medium_byte_string), medium_byte_string); - - EXPECT_EQ(ByteString(small_byte_string), small_byte_string); - EXPECT_EQ(ByteString(medium_byte_string), medium_byte_string); -} - -TEST_P(ByteStringTest, MoveConstruct) { - const auto& small_byte_string = [this]() { - return ByteString(GetAllocator(), GetSmallStringView()); - }; - const auto& medium_byte_string = [this]() { - return ByteString(GetAllocator(), GetMediumStringView()); - }; - const auto& large_byte_string = [this]() { - return ByteString(GetAllocator(), GetMediumOrLargeCord()); - }; - - EXPECT_EQ(ByteString(NewDeleteAllocator(), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(NewDeleteAllocator(), medium_byte_string()), - medium_byte_string()); - EXPECT_EQ(ByteString(NewDeleteAllocator(), large_byte_string()), - large_byte_string()); - - google::protobuf::Arena arena; - EXPECT_EQ(ByteString(ArenaAllocator(&arena), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), medium_byte_string()), - medium_byte_string()); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), large_byte_string()), - large_byte_string()); - - EXPECT_EQ(ByteString(GetAllocator(), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(GetAllocator(), medium_byte_string()), - medium_byte_string()); - EXPECT_EQ(ByteString(GetAllocator(), large_byte_string()), - large_byte_string()); - - EXPECT_EQ(ByteString(small_byte_string()), small_byte_string()); - EXPECT_EQ(ByteString(medium_byte_string()), medium_byte_string()); - EXPECT_EQ(ByteString(large_byte_string()), large_byte_string()); -} - -TEST_P(ByteStringTest, MoveConstructFromExternal) { - const auto& small_byte_string = []() { - return ByteString::FromExternal(GetSmallStringView()); - }; - const auto& medium_byte_string = []() { - return ByteString::FromExternal(GetMediumStringView()); - }; - - EXPECT_EQ(ByteString(NewDeleteAllocator(), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(NewDeleteAllocator(), medium_byte_string()), - medium_byte_string()); - - google::protobuf::Arena arena; - EXPECT_EQ(ByteString(ArenaAllocator(&arena), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(ArenaAllocator(&arena), medium_byte_string()), - medium_byte_string()); - - EXPECT_EQ(ByteString(GetAllocator(), small_byte_string()), - small_byte_string()); - EXPECT_EQ(ByteString(GetAllocator(), medium_byte_string()), - medium_byte_string()); - - EXPECT_EQ(ByteString(small_byte_string()), small_byte_string()); - EXPECT_EQ(ByteString(medium_byte_string()), medium_byte_string()); -} - -TEST_P(ByteStringTest, CopyFromByteString) { - ByteString small_byte_string = - ByteString(GetAllocator(), GetSmallStringView()); - ByteString medium_byte_string = - ByteString(GetAllocator(), GetMediumStringView()); - ByteString large_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); - - ByteString new_delete_byte_string(NewDeleteAllocator<>{}); - // Small <= Small - new_delete_byte_string = small_byte_string; - EXPECT_EQ(new_delete_byte_string, small_byte_string); - // Small <= Medium - new_delete_byte_string = medium_byte_string; - EXPECT_EQ(new_delete_byte_string, medium_byte_string); - // Medium <= Medium - new_delete_byte_string = medium_byte_string; - EXPECT_EQ(new_delete_byte_string, medium_byte_string); - // Medium <= Large - new_delete_byte_string = large_byte_string; - EXPECT_EQ(new_delete_byte_string, large_byte_string); - // Large <= Large - new_delete_byte_string = large_byte_string; - EXPECT_EQ(new_delete_byte_string, large_byte_string); - // Large <= Small - new_delete_byte_string = small_byte_string; - EXPECT_EQ(new_delete_byte_string, small_byte_string); - // Small <= Large - new_delete_byte_string = large_byte_string; - EXPECT_EQ(new_delete_byte_string, large_byte_string); - // Large <= Medium - new_delete_byte_string = medium_byte_string; - EXPECT_EQ(new_delete_byte_string, medium_byte_string); - // Medium <= Small - new_delete_byte_string = small_byte_string; - EXPECT_EQ(new_delete_byte_string, small_byte_string); - - google::protobuf::Arena arena; - ByteString arena_byte_string(ArenaAllocator<>{&arena}); - // Small <= Small - arena_byte_string = small_byte_string; - EXPECT_EQ(arena_byte_string, small_byte_string); - // Small <= Medium - arena_byte_string = medium_byte_string; - EXPECT_EQ(arena_byte_string, medium_byte_string); - // Medium <= Medium - arena_byte_string = medium_byte_string; - EXPECT_EQ(arena_byte_string, medium_byte_string); - // Medium <= Large - arena_byte_string = large_byte_string; - EXPECT_EQ(arena_byte_string, large_byte_string); - // Large <= Large - arena_byte_string = large_byte_string; - EXPECT_EQ(arena_byte_string, large_byte_string); - // Large <= Small - arena_byte_string = small_byte_string; - EXPECT_EQ(arena_byte_string, small_byte_string); - // Small <= Large - arena_byte_string = large_byte_string; - EXPECT_EQ(arena_byte_string, large_byte_string); - // Large <= Medium - arena_byte_string = medium_byte_string; - EXPECT_EQ(arena_byte_string, medium_byte_string); - // Medium <= Small - arena_byte_string = small_byte_string; - EXPECT_EQ(arena_byte_string, small_byte_string); - - ByteString allocator_byte_string(GetAllocator()); - // Small <= Small - allocator_byte_string = small_byte_string; - EXPECT_EQ(allocator_byte_string, small_byte_string); - // Small <= Medium - allocator_byte_string = medium_byte_string; - EXPECT_EQ(allocator_byte_string, medium_byte_string); - // Medium <= Medium - allocator_byte_string = medium_byte_string; - EXPECT_EQ(allocator_byte_string, medium_byte_string); - // Medium <= Large - allocator_byte_string = large_byte_string; - EXPECT_EQ(allocator_byte_string, large_byte_string); - // Large <= Large - allocator_byte_string = large_byte_string; - EXPECT_EQ(allocator_byte_string, large_byte_string); - // Large <= Small - allocator_byte_string = small_byte_string; - EXPECT_EQ(allocator_byte_string, small_byte_string); - // Small <= Large - allocator_byte_string = large_byte_string; - EXPECT_EQ(allocator_byte_string, large_byte_string); - // Large <= Medium - allocator_byte_string = medium_byte_string; - EXPECT_EQ(allocator_byte_string, medium_byte_string); - // Medium <= Small - allocator_byte_string = small_byte_string; - EXPECT_EQ(allocator_byte_string, small_byte_string); - - // Miscellaneous cases not covered above. - // Large <= Medium Arena String - ByteString large_new_delete_byte_string(NewDeleteAllocator<>{}, - GetMediumOrLargeCord()); - ByteString medium_arena_byte_string(ArenaAllocator<>{&arena}, - GetMediumStringView()); - large_new_delete_byte_string = medium_arena_byte_string; - EXPECT_EQ(large_new_delete_byte_string, medium_arena_byte_string); -} - -TEST_P(ByteStringTest, MoveFrom) { - const auto& small_byte_string = [this]() { - return ByteString(GetAllocator(), GetSmallStringView()); - }; - const auto& medium_byte_string = [this]() { - return ByteString(GetAllocator(), GetMediumStringView()); - }; - const auto& large_byte_string = [this]() { - return ByteString(GetAllocator(), GetMediumOrLargeCord()); - }; - - ByteString new_delete_byte_string(NewDeleteAllocator<>{}); - // Small <= Small - new_delete_byte_string = small_byte_string(); - EXPECT_EQ(new_delete_byte_string, small_byte_string()); - // Small <= Medium - new_delete_byte_string = medium_byte_string(); - EXPECT_EQ(new_delete_byte_string, medium_byte_string()); - // Medium <= Medium - new_delete_byte_string = medium_byte_string(); - EXPECT_EQ(new_delete_byte_string, medium_byte_string()); - // Medium <= Large - new_delete_byte_string = large_byte_string(); - EXPECT_EQ(new_delete_byte_string, large_byte_string()); - // Large <= Large - new_delete_byte_string = large_byte_string(); - EXPECT_EQ(new_delete_byte_string, large_byte_string()); - // Large <= Small - new_delete_byte_string = small_byte_string(); - EXPECT_EQ(new_delete_byte_string, small_byte_string()); - // Small <= Large - new_delete_byte_string = large_byte_string(); - EXPECT_EQ(new_delete_byte_string, large_byte_string()); - // Large <= Medium - new_delete_byte_string = medium_byte_string(); - EXPECT_EQ(new_delete_byte_string, medium_byte_string()); - // Medium <= Small - new_delete_byte_string = small_byte_string(); - EXPECT_EQ(new_delete_byte_string, small_byte_string()); - - google::protobuf::Arena arena; - ByteString arena_byte_string(ArenaAllocator<>{&arena}); - // Small <= Small - arena_byte_string = small_byte_string(); - EXPECT_EQ(arena_byte_string, small_byte_string()); - // Small <= Medium - arena_byte_string = medium_byte_string(); - EXPECT_EQ(arena_byte_string, medium_byte_string()); - // Medium <= Medium - arena_byte_string = medium_byte_string(); - EXPECT_EQ(arena_byte_string, medium_byte_string()); - // Medium <= Large - arena_byte_string = large_byte_string(); - EXPECT_EQ(arena_byte_string, large_byte_string()); - // Large <= Large - arena_byte_string = large_byte_string(); - EXPECT_EQ(arena_byte_string, large_byte_string()); - // Large <= Small - arena_byte_string = small_byte_string(); - EXPECT_EQ(arena_byte_string, small_byte_string()); - // Small <= Large - arena_byte_string = large_byte_string(); - EXPECT_EQ(arena_byte_string, large_byte_string()); - // Large <= Medium - arena_byte_string = medium_byte_string(); - EXPECT_EQ(arena_byte_string, medium_byte_string()); - // Medium <= Small - arena_byte_string = small_byte_string(); - EXPECT_EQ(arena_byte_string, small_byte_string()); - - ByteString allocator_byte_string(GetAllocator()); - // Small <= Small - allocator_byte_string = small_byte_string(); - EXPECT_EQ(allocator_byte_string, small_byte_string()); - // Small <= Medium - allocator_byte_string = medium_byte_string(); - EXPECT_EQ(allocator_byte_string, medium_byte_string()); - // Medium <= Medium - allocator_byte_string = medium_byte_string(); - EXPECT_EQ(allocator_byte_string, medium_byte_string()); - // Medium <= Large - allocator_byte_string = large_byte_string(); - EXPECT_EQ(allocator_byte_string, large_byte_string()); - // Large <= Large - allocator_byte_string = large_byte_string(); - EXPECT_EQ(allocator_byte_string, large_byte_string()); - // Large <= Small - allocator_byte_string = small_byte_string(); - EXPECT_EQ(allocator_byte_string, small_byte_string()); - // Small <= Large - allocator_byte_string = large_byte_string(); - EXPECT_EQ(allocator_byte_string, large_byte_string()); - // Large <= Medium - allocator_byte_string = medium_byte_string(); - EXPECT_EQ(allocator_byte_string, medium_byte_string()); - // Medium <= Small - allocator_byte_string = small_byte_string(); - EXPECT_EQ(allocator_byte_string, small_byte_string()); - - // Miscellaneous cases not covered above. - // Large <= Medium Arena String - ByteString large_new_delete_byte_string(NewDeleteAllocator<>{}, - GetMediumOrLargeCord()); - ByteString medium_arena_byte_string(ArenaAllocator<>{&arena}, - GetMediumStringView()); - large_new_delete_byte_string = std::move(medium_arena_byte_string); - EXPECT_EQ(large_new_delete_byte_string, GetMediumStringView()); -} - -TEST_P(ByteStringTest, Swap) { - using std::swap; - ByteString empty_byte_string(GetAllocator()); - ByteString small_byte_string = - ByteString(GetAllocator(), GetSmallStringView()); - ByteString medium_byte_string = - ByteString(GetAllocator(), GetMediumStringView()); - ByteString large_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); - - // Small <=> Small - swap(empty_byte_string, small_byte_string); - EXPECT_EQ(empty_byte_string, GetSmallStringView()); - EXPECT_EQ(small_byte_string, ""); - swap(empty_byte_string, small_byte_string); - EXPECT_EQ(empty_byte_string, ""); - EXPECT_EQ(small_byte_string, GetSmallStringView()); - - // Small <=> Medium - swap(small_byte_string, medium_byte_string); - EXPECT_EQ(small_byte_string, GetMediumStringView()); - EXPECT_EQ(medium_byte_string, GetSmallStringView()); - swap(small_byte_string, medium_byte_string); - EXPECT_EQ(small_byte_string, GetSmallStringView()); - EXPECT_EQ(medium_byte_string, GetMediumStringView()); - - // Small <=> Large - swap(small_byte_string, large_byte_string); - EXPECT_EQ(small_byte_string, GetMediumOrLargeCord()); - EXPECT_EQ(large_byte_string, GetSmallStringView()); - swap(small_byte_string, large_byte_string); - EXPECT_EQ(small_byte_string, GetSmallStringView()); - EXPECT_EQ(large_byte_string, GetMediumOrLargeCord()); - - // Medium <=> Medium - static constexpr absl::string_view kDifferentMediumStringView = - "A different string that is too large for the small string optimization!"; - ByteString other_medium_byte_string = - ByteString(GetAllocator(), kDifferentMediumStringView); - swap(medium_byte_string, other_medium_byte_string); - EXPECT_EQ(medium_byte_string, kDifferentMediumStringView); - EXPECT_EQ(other_medium_byte_string, GetMediumStringView()); - swap(medium_byte_string, other_medium_byte_string); - EXPECT_EQ(medium_byte_string, GetMediumStringView()); - EXPECT_EQ(other_medium_byte_string, kDifferentMediumStringView); - - // Medium <=> Large - swap(medium_byte_string, large_byte_string); - EXPECT_EQ(medium_byte_string, GetMediumOrLargeCord()); - EXPECT_EQ(large_byte_string, GetMediumStringView()); - swap(medium_byte_string, large_byte_string); - EXPECT_EQ(medium_byte_string, GetMediumStringView()); - EXPECT_EQ(large_byte_string, GetMediumOrLargeCord()); - - // Large <=> Large - const absl::Cord different_medium_or_large_cord = - absl::Cord(kDifferentMediumStringView); - ByteString other_large_byte_string = - ByteString(GetAllocator(), different_medium_or_large_cord); - swap(large_byte_string, other_large_byte_string); - EXPECT_EQ(large_byte_string, different_medium_or_large_cord); - EXPECT_EQ(other_large_byte_string, GetMediumStringView()); - swap(large_byte_string, other_large_byte_string); - EXPECT_EQ(large_byte_string, GetMediumStringView()); - EXPECT_EQ(other_large_byte_string, different_medium_or_large_cord); - - // Miscellaneous cases not covered above. These do not swap a second time to - // restore state, so they are destructive. - // Small <=> Different Allocator Medium - ByteString medium_new_delete_byte_string = - ByteString(NewDeleteAllocator<>{}, kDifferentMediumStringView); - swap(empty_byte_string, medium_new_delete_byte_string); - EXPECT_EQ(empty_byte_string, kDifferentMediumStringView); - EXPECT_EQ(medium_new_delete_byte_string, ""); - // Small <=> Different Allocator Large - ByteString large_new_delete_byte_string = - ByteString(NewDeleteAllocator<>{}, GetMediumOrLargeCord()); - swap(small_byte_string, large_new_delete_byte_string); - EXPECT_EQ(small_byte_string, GetMediumOrLargeCord()); - EXPECT_EQ(large_new_delete_byte_string, GetSmallStringView()); - // Medium <=> Different Allocator Large - large_new_delete_byte_string = - ByteString(NewDeleteAllocator<>{}, different_medium_or_large_cord); - swap(medium_byte_string, large_new_delete_byte_string); - EXPECT_EQ(medium_byte_string, different_medium_or_large_cord); - EXPECT_EQ(large_new_delete_byte_string, GetMediumStringView()); - // Medium <=> Different Allocator Medium - medium_byte_string = ByteString(GetAllocator(), GetMediumStringView()); - medium_new_delete_byte_string = - ByteString(NewDeleteAllocator<>{}, kDifferentMediumStringView); - swap(medium_byte_string, medium_new_delete_byte_string); - EXPECT_EQ(medium_byte_string, kDifferentMediumStringView); - EXPECT_EQ(medium_new_delete_byte_string, GetMediumStringView()); -} - -TEST_P(ByteStringTest, FlattenSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.Flatten(), GetSmallStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); -} - -TEST_P(ByteStringTest, FlattenMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); - EXPECT_EQ(byte_string.Flatten(), GetMediumStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); -} - -TEST_P(ByteStringTest, FlattenLarge) { - if (GetAllocator().arena() != nullptr) { - GTEST_SKIP(); - } - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); - EXPECT_EQ(byte_string.Flatten(), GetMediumStringView()); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); -} - -TEST_P(ByteStringTest, TryFlatSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, TryFlatSmall) { + ByteString byte_string = ByteString::From(GetSmallStringView(), GetArena()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); EXPECT_THAT(byte_string.TryFlat(), Optional(GetSmallStringView())); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); } -TEST_P(ByteStringTest, TryFlatMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, TryFlatMedium) { + ByteString byte_string = ByteString::From(GetMediumStringView(), GetArena()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); EXPECT_THAT(byte_string.TryFlat(), Optional(GetMediumStringView())); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); } -TEST_P(ByteStringTest, TryFlatLarge) { - if (GetAllocator().arena() != nullptr) { +TEST_F(ByteStringTest, TryFlatLarge) { + if (GetArena() != nullptr) { GTEST_SKIP(); } ByteString byte_string = - ByteString(GetAllocator(), GetMediumOrLargeFragmentedCord()); + ByteString::From(GetMediumOrLargeFragmentedCord(), GetArena()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); EXPECT_THAT(byte_string.TryFlat(), Eq(std::nullopt)); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); } -TEST_P(ByteStringTest, Equals) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, Equals) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_TRUE(byte_string.Equals(GetMediumStringView())); } -TEST_P(ByteStringTest, Compare) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, Compare) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_EQ(byte_string.Compare(GetMediumStringView()), 0); EXPECT_EQ(byte_string.Compare(GetMediumOrLargeCord()), 0); } -TEST_P(ByteStringTest, StartsWith) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, StartsWith) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_TRUE(byte_string.StartsWith( GetMediumStringView().substr(0, kSmallByteStringCapacity))); EXPECT_TRUE(byte_string.StartsWith( GetMediumOrLargeCord().Subcord(0, kSmallByteStringCapacity))); } -TEST_P(ByteStringTest, EndsWith) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, EndsWith) { + ByteString byte_string = ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_TRUE(byte_string.EndsWith( GetMediumStringView().substr(kSmallByteStringCapacity))); EXPECT_TRUE(byte_string.EndsWith(GetMediumOrLargeCord().Subcord( @@ -804,8 +271,8 @@ TEST_P(ByteStringTest, EndsWith) { GetMediumOrLargeCord().size() - kSmallByteStringCapacity))); } -TEST_P(ByteStringTest, Find) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, Find) { + ByteString byte_string = ByteString::From(GetMediumStringView(), GetArena()); // Find string_view EXPECT_THAT(byte_string.Find("A string"), Optional(0)); @@ -833,14 +300,14 @@ TEST_P(ByteStringTest, Find) { EXPECT_THAT(byte_string.Find(absl::Cord(""), 3), Optional(3)); } -TEST_P(ByteStringTest, FindEdgeCases) { - ByteString empty_byte_string(GetAllocator(), ""); +TEST_F(ByteStringTest, FindEdgeCases) { + ByteString empty_byte_string; EXPECT_THAT(empty_byte_string.Find("a"), Eq(std::nullopt)); EXPECT_THAT(empty_byte_string.Find(""), Optional(0)); ByteString cord_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString::From(GetMediumOrLargeCord(), GetArena()); EXPECT_THAT(cord_byte_string.Find("not found"), Eq(std::nullopt)); - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); + ByteString byte_string = ByteString::From(GetMediumStringView(), GetArena()); // Needle longer than haystack. EXPECT_THAT(byte_string.Find(std::string(byte_string.size() + 1, 'a')), @@ -875,13 +342,13 @@ TEST_P(ByteStringTest, FindEdgeCases) { // Search with fragmented cord needle on string_view backed ByteString with // partial match. - ByteString partial_match_haystack(GetAllocator(), "abababac"); + ByteString partial_match_haystack = ByteString::WrapUnsafe("abababac"); absl::Cord partial_match_needle = absl::MakeFragmentedCord({"aba", "c"}); EXPECT_THAT(partial_match_haystack.Find(partial_match_needle), Optional(4)); // Search with fragmented cord needle where first chunk is found but not // enough space for the rest. - ByteString short_haystack(GetAllocator(), "abcdefg"); + ByteString short_haystack = ByteString::WrapUnsafe("abcdefg"); absl::Cord needle_too_long = absl::MakeFragmentedCord({"ef", "gh"}); EXPECT_THAT(short_haystack.Find(needle_too_long), Eq(std::nullopt)); @@ -891,8 +358,8 @@ TEST_P(ByteStringTest, FindEdgeCases) { EXPECT_THAT(byte_string.Find(fragmented_empty_cord, 3), Optional(3)); // Search for suffix in a fragmented cord. - ByteString fragmented_cord_byte_string(GetAllocator(), - GetMediumOrLargeFragmentedCord()); + ByteString fragmented_cord_byte_string = + ByteString::From(GetMediumOrLargeFragmentedCord(), GetArena()); EXPECT_THAT(fragmented_cord_byte_string.Find(suffix_sv), Optional(fragmented_cord_byte_string.size() - suffix_sv.size())); EXPECT_THAT(fragmented_cord_byte_string.Find(absl::Cord(suffix_sv)), @@ -900,311 +367,305 @@ TEST_P(ByteStringTest, FindEdgeCases) { } #ifndef NDEBUG -TEST_P(ByteStringTest, FindOutOfBounds) { - ByteString byte_string = ByteString(GetAllocator(), "test"); +TEST_F(ByteStringTest, FindOutOfBounds) { + ByteString byte_string = ByteString::WrapUnsafe("test"); EXPECT_DEATH(byte_string.Find("t", 5), _); } #endif -TEST_P(ByteStringTest, Substring) { +TEST_F(ByteStringTest, Substring) { // small byte_string substring - ByteString small_byte_string = - ByteString(GetAllocator(), GetSmallStringView()); + ByteString small_byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(small_byte_string.Substring(1, 5), GetSmallStringView().substr(1, 4)); EXPECT_EQ(small_byte_string.Substring(0, small_byte_string.size()), GetSmallStringView()); EXPECT_EQ(small_byte_string.Substring(1, 1), ""); // medium byte_string substring - ByteString medium_byte_string = - ByteString(GetAllocator(), GetMediumStringView()); + ByteString medium_byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(medium_byte_string.Substring(2, 12), GetMediumStringView().substr(2, 10)); EXPECT_EQ(medium_byte_string.Substring(0, medium_byte_string.size()), GetMediumStringView()); // large byte_string substring ByteString large_byte_string = - ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(large_byte_string.Substring(3, 15), GetMediumOrLargeCord().Subcord(3, 12)); EXPECT_EQ(large_byte_string.Substring(0, large_byte_string.size()), GetMediumOrLargeCord()); // substring with one parameter - ByteString tacocat_byte_string = ByteString(GetAllocator(), "tacocat"); + ByteString tacocat_byte_string = ByteString::WrapUnsafe("tacocat"); EXPECT_EQ(tacocat_byte_string.Substring(4), "cat"); } -TEST_P(ByteStringTest, SubstringEdgeCases) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, SubstringEdgeCases) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.Substring(byte_string.size(), byte_string.size()), ""); EXPECT_EQ(byte_string.Substring(0, 0), ""); } #ifndef NDEBUG -TEST_P(ByteStringTest, SubstringOutOfBounds) { - ByteString byte_string = ByteString(GetAllocator(), "test"); +TEST_F(ByteStringTest, SubstringOutOfBounds) { + ByteString byte_string = ByteString::WrapUnsafe("test"); EXPECT_DEATH(static_cast(byte_string.Substring(5, 5)), _); EXPECT_DEATH(static_cast(byte_string.Substring(0, 5)), _); EXPECT_DEATH(static_cast(byte_string.Substring(3, 2)), _); } #endif -TEST_P(ByteStringTest, RemovePrefixSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, RemovePrefixSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); byte_string.RemovePrefix(1); EXPECT_EQ(byte_string, GetSmallStringView().substr(1)); } -TEST_P(ByteStringTest, RemovePrefixMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, RemovePrefixMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); byte_string.RemovePrefix(byte_string.size() - kSmallByteStringCapacity); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); EXPECT_EQ(byte_string, GetMediumStringView().substr(GetMediumStringView().size() - kSmallByteStringCapacity)); } -TEST_P(ByteStringTest, RemovePrefixMediumOrLarge) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, RemovePrefixMediumOrLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); byte_string.RemovePrefix(byte_string.size() - kSmallByteStringCapacity); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); EXPECT_EQ(byte_string, GetMediumStringView().substr(GetMediumStringView().size() - kSmallByteStringCapacity)); } -TEST_P(ByteStringTest, RemoveSuffixSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, RemoveSuffixSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); byte_string.RemoveSuffix(1); EXPECT_EQ(byte_string, GetSmallStringView().substr(0, GetSmallStringView().size() - 1)); } -TEST_P(ByteStringTest, RemoveSuffixMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, RemoveSuffixMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); byte_string.RemoveSuffix(byte_string.size() - kSmallByteStringCapacity); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kMedium); EXPECT_EQ(byte_string, GetMediumStringView().substr(0, kSmallByteStringCapacity)); } -TEST_P(ByteStringTest, RemoveSuffixMediumOrLarge) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, RemoveSuffixMediumOrLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); byte_string.RemoveSuffix(byte_string.size() - kSmallByteStringCapacity); - EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); + EXPECT_EQ(GetKind(byte_string), ByteStringKind::kLarge); EXPECT_EQ(byte_string, GetMediumStringView().substr(0, kSmallByteStringCapacity)); } -TEST_P(ByteStringTest, ToStringSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, ToStringSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.ToString(), byte_string); } -TEST_P(ByteStringTest, ToStringMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, ToStringMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.ToString(), byte_string); } -TEST_P(ByteStringTest, ToStringLarge) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, ToStringLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(byte_string.ToString(), byte_string); } -TEST_P(ByteStringTest, ToStringViewSmall) { +TEST_F(ByteStringTest, ToStringViewSmall) { std::string scratch; - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.ToStringView(&scratch), GetSmallStringView()); } -TEST_P(ByteStringTest, ToStringViewMedium) { +TEST_F(ByteStringTest, ToStringViewMedium) { std::string scratch; - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.ToStringView(&scratch), GetMediumStringView()); } -TEST_P(ByteStringTest, ToStringViewLarge) { +TEST_F(ByteStringTest, ToStringViewLarge) { std::string scratch; - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(byte_string.ToStringView(&scratch), GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, AsStringViewSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, AsStringViewSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.AsStringView(), GetSmallStringView()); } -TEST_P(ByteStringTest, AsStringViewMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, AsStringViewMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.AsStringView(), GetMediumStringView()); } -TEST_P(ByteStringTest, AsStringViewLarge) { - ByteString byte_string = ByteString(GetMediumOrLargeCord()); +TEST_F(ByteStringTest, AsStringViewLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_DEATH(byte_string.AsStringView(), _); } -TEST_P(ByteStringTest, CopyToStringSmall) { +TEST_F(ByteStringTest, CopyToStringSmall) { std::string out; - ByteString(GetAllocator(), GetSmallStringView()).CopyToString(&out); + ByteString::WrapUnsafe(GetSmallStringView()).CopyToString(&out); EXPECT_EQ(out, GetSmallStringView()); } -TEST_P(ByteStringTest, CopyToStringMedium) { +TEST_F(ByteStringTest, CopyToStringMedium) { std::string out; - ByteString(GetAllocator(), GetMediumStringView()).CopyToString(&out); + ByteString::WrapUnsafe(GetMediumStringView()).CopyToString(&out); EXPECT_EQ(out, GetMediumStringView()); } -TEST_P(ByteStringTest, CopyToStringLarge) { +TEST_F(ByteStringTest, CopyToStringLarge) { std::string out; - ByteString(GetAllocator(), GetMediumOrLargeCord()).CopyToString(&out); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()).CopyToString(&out); EXPECT_EQ(out, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, AppendToStringSmall) { +TEST_F(ByteStringTest, AppendToStringSmall) { std::string out; - ByteString(GetAllocator(), GetSmallStringView()).AppendToString(&out); + ByteString::WrapUnsafe(GetSmallStringView()).AppendToString(&out); EXPECT_EQ(out, GetSmallStringView()); } -TEST_P(ByteStringTest, AppendToStringMedium) { +TEST_F(ByteStringTest, AppendToStringMedium) { std::string out; - ByteString(GetAllocator(), GetMediumStringView()).AppendToString(&out); + ByteString::WrapUnsafe(GetMediumStringView()).AppendToString(&out); EXPECT_EQ(out, GetMediumStringView()); } -TEST_P(ByteStringTest, AppendToStringLarge) { +TEST_F(ByteStringTest, AppendToStringLarge) { std::string out; - ByteString(GetAllocator(), GetMediumOrLargeCord()).AppendToString(&out); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()).AppendToString(&out); EXPECT_EQ(out, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, ToCordSmall) { - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); +TEST_F(ByteStringTest, ToCordSmall) { + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.ToCord(), byte_string); EXPECT_EQ(std::move(byte_string).ToCord(), GetSmallStringView()); } -TEST_P(ByteStringTest, ToCordMedium) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); +TEST_F(ByteStringTest, ToCordMedium) { + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.ToCord(), byte_string); EXPECT_EQ(std::move(byte_string).ToCord(), GetMediumStringView()); } -TEST_P(ByteStringTest, ToCordLarge) { - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); +TEST_F(ByteStringTest, ToCordLarge) { + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(byte_string.ToCord(), byte_string); EXPECT_EQ(std::move(byte_string).ToCord(), GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, CopyToCordSmall) { +TEST_F(ByteStringTest, CopyToCordSmall) { absl::Cord out; - ByteString(GetAllocator(), GetSmallStringView()).CopyToCord(&out); + ByteString::WrapUnsafe(GetSmallStringView()).CopyToCord(&out); EXPECT_EQ(out, GetSmallStringView()); } -TEST_P(ByteStringTest, CopyToCordMedium) { +TEST_F(ByteStringTest, CopyToCordMedium) { absl::Cord out; - ByteString(GetAllocator(), GetMediumStringView()).CopyToCord(&out); + ByteString::WrapUnsafe(GetMediumStringView()).CopyToCord(&out); EXPECT_EQ(out, GetMediumStringView()); } -TEST_P(ByteStringTest, CopyToCordLarge) { +TEST_F(ByteStringTest, CopyToCordLarge) { absl::Cord out; - ByteString(GetAllocator(), GetMediumOrLargeCord()).CopyToCord(&out); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()).CopyToCord(&out); EXPECT_EQ(out, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, AppendToCordSmall) { +TEST_F(ByteStringTest, AppendToCordSmall) { absl::Cord out; - ByteString(GetAllocator(), GetSmallStringView()).AppendToCord(&out); + ByteString::WrapUnsafe(GetSmallStringView()).AppendToCord(&out); EXPECT_EQ(out, GetSmallStringView()); } -TEST_P(ByteStringTest, AppendToCordMedium) { +TEST_F(ByteStringTest, AppendToCordMedium) { absl::Cord out; - ByteString(GetAllocator(), GetMediumStringView()).AppendToCord(&out); + ByteString::WrapUnsafe(GetMediumStringView()).AppendToCord(&out); EXPECT_EQ(out, GetMediumStringView()); } -TEST_P(ByteStringTest, AppendToCordLarge) { +TEST_F(ByteStringTest, AppendToCordLarge) { absl::Cord out; - ByteString(GetAllocator(), GetMediumOrLargeCord()).AppendToCord(&out); + ByteString::WrapUnsafe(&GetMediumOrLargeCord()).AppendToCord(&out); EXPECT_EQ(out, GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, CloneSmall) { +TEST_F(ByteStringTest, CloneSmall) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(byte_string.Clone(&arena), byte_string); } -TEST_P(ByteStringTest, CloneMedium) { +TEST_F(ByteStringTest, CloneMedium) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(byte_string.Clone(&arena), byte_string); } -TEST_P(ByteStringTest, CloneLarge) { +TEST_F(ByteStringTest, CloneLarge) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(byte_string.Clone(&arena), byte_string); } -TEST_P(ByteStringTest, LegacyByteStringSmall) { +TEST_F(ByteStringTest, LegacyByteStringSmall) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetSmallStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/false, &arena), GetSmallStringView()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/true, &arena), GetSmallStringView()); } -TEST_P(ByteStringTest, LegacyByteStringMedium) { +TEST_F(ByteStringTest, LegacyByteStringMedium) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetMediumStringView()); + ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/false, &arena), GetMediumStringView()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/true, &arena), GetMediumStringView()); } -TEST_P(ByteStringTest, LegacyByteStringLarge) { +TEST_F(ByteStringTest, LegacyByteStringLarge) { google::protobuf::Arena arena; - ByteString byte_string = ByteString(GetAllocator(), GetMediumOrLargeCord()); + ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/false, &arena), GetMediumOrLargeCord()); EXPECT_EQ(LegacyByteString(byte_string, /*stable=*/true, &arena), GetMediumOrLargeCord()); } -TEST_P(ByteStringTest, HashValue) { - EXPECT_EQ(absl::HashOf(ByteString(GetAllocator(), GetSmallStringView())), +TEST_F(ByteStringTest, HashValue) { + EXPECT_EQ(absl::HashOf(ByteString::WrapUnsafe(GetSmallStringView())), absl::HashOf(GetSmallStringView())); - EXPECT_EQ(absl::HashOf(ByteString(GetAllocator(), GetMediumStringView())), + EXPECT_EQ(absl::HashOf(ByteString::WrapUnsafe(GetMediumStringView())), absl::HashOf(GetMediumStringView())); - EXPECT_EQ(absl::HashOf(ByteString(GetAllocator(), GetMediumOrLargeCord())), + EXPECT_EQ(absl::HashOf(ByteString::WrapUnsafe(&GetMediumOrLargeCord())), absl::HashOf(GetMediumOrLargeCord())); } -INSTANTIATE_TEST_SUITE_P(ByteStringTest, ByteStringTest, - ::testing::Values(AllocatorKind::kNewDelete, - AllocatorKind::kArena)); - } // namespace } // namespace cel::common_internal diff --git a/common/internal/value_conversion.cc b/common/internal/value_conversion.cc index 57cf2224b..ef4dda231 100644 --- a/common/internal/value_conversion.cc +++ b/common/internal/value_conversion.cc @@ -224,9 +224,9 @@ absl::StatusOr FromExprValue( case ExprValueKind::kDoubleValue: return cel::DoubleValue(value.double_value()); case ExprValueKind::kStringValue: - return cel::StringValue(value.string_value()); + return cel::StringValue::From(value.string_value(), arena); case ExprValueKind::kBytesValue: - return cel::BytesValue(value.bytes_value()); + return cel::BytesValue::From(value.bytes_value(), arena); case ExprValueKind::kNullValue: return cel::NullValue(); case ExprValueKind::kObjectValue: diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 59f9db278..c6f00d519 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -1044,12 +1044,10 @@ absl::Status ModernValue(google::protobuf::Arena* arena, result = DoubleValue{legacy_value.DoubleOrDie()}; return absl::OkStatus(); case CelValue::Type::kString: - result = StringValue(Borrower::Arena(arena), - legacy_value.StringOrDie().value()); + result = StringValue::Wrap(legacy_value.StringOrDie().value(), nullptr); return absl::OkStatus(); case CelValue::Type::kBytes: - result = - BytesValue(Borrower::Arena(arena), legacy_value.BytesOrDie().value()); + result = BytesValue::Wrap(legacy_value.BytesOrDie().value(), nullptr); return absl::OkStatus(); case CelValue::Type::kMessage: { auto message_wrapper = legacy_value.MessageWrapperOrDie(); @@ -1166,11 +1164,9 @@ absl::StatusOr FromLegacyValue(google::protobuf::Arena* arena, case CelValue::Type::kDouble: return DoubleValue(legacy_value.DoubleOrDie()); case CelValue::Type::kString: - return StringValue(Borrower::Arena(arena), - legacy_value.StringOrDie().value()); + return StringValue::Wrap(legacy_value.StringOrDie().value(), nullptr); case CelValue::Type::kBytes: - return BytesValue(Borrower::Arena(arena), - legacy_value.BytesOrDie().value()); + return BytesValue::Wrap(legacy_value.BytesOrDie().value(), nullptr); case CelValue::Type::kMessage: { auto message_wrapper = legacy_value.MessageWrapperOrDie(); return common_internal::MakeLegacyStructValue( diff --git a/common/type_reflector_test.cc b/common/type_reflector_test.cc index d9c855e4b..961de0d8e 100644 --- a/common/type_reflector_test.cc +++ b/common/type_reflector_test.cc @@ -32,6 +32,7 @@ namespace cel { namespace { +using ::absl_testing::IsOk; using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::cel::test::ErrorValueIs; @@ -178,8 +179,8 @@ TEST_F(TypeReflectorTest, NewMapValueBuilderCoverage_DynamicDynamic) { EXPECT_OK(builder->Put(IntValue(1), IntValue(4))); EXPECT_OK(builder->Put(UintValue(0), IntValue(5))); EXPECT_OK(builder->Put(UintValue(1), IntValue(6))); - EXPECT_OK(builder->Put(StringValue("a"), IntValue(7))); - EXPECT_OK(builder->Put(StringValue("b"), IntValue(8))); + EXPECT_THAT(builder->Put(StringValue::WrapUnsafe("a"), IntValue(7)), IsOk()); + EXPECT_THAT(builder->Put(StringValue::WrapUnsafe("b"), IntValue(8)), IsOk()); EXPECT_EQ(builder->Size(), 8); EXPECT_FALSE(builder->IsEmpty()); auto value = std::move(*builder).Build(); @@ -407,17 +408,18 @@ TEST_F(TypeReflectorTest, NewValueBuilder_StringValue) { arena(), internal::GetTestingDescriptorPool(), internal::GetTestingMessageFactory(), "google.protobuf.StringValue"); ASSERT_THAT(builder, NotNull()); - EXPECT_THAT(builder->SetFieldByName("value", StringValue("foo")), + EXPECT_THAT(builder->SetFieldByName("value", StringValue::WrapUnsafe("foo")), IsOkAndHolds(Eq(std::nullopt))); - EXPECT_THAT(builder->SetFieldByName("does_not_exist", StringValue("foo")), - IsOkAndHolds(Optional( - ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); + EXPECT_THAT( + builder->SetFieldByName("does_not_exist", StringValue::WrapUnsafe("foo")), + IsOkAndHolds( + Optional(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); - EXPECT_THAT(builder->SetFieldByNumber(1, StringValue("foo")), + EXPECT_THAT(builder->SetFieldByNumber(1, StringValue::WrapUnsafe("foo")), IsOkAndHolds(Eq(std::nullopt))); - EXPECT_THAT(builder->SetFieldByNumber(2, StringValue("foo")), + EXPECT_THAT(builder->SetFieldByNumber(2, StringValue::WrapUnsafe("foo")), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); EXPECT_THAT(builder->SetFieldByNumber(1, BoolValue(true)), @@ -433,17 +435,18 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BytesValue) { arena(), internal::GetTestingDescriptorPool(), internal::GetTestingMessageFactory(), "google.protobuf.BytesValue"); ASSERT_THAT(builder, NotNull()); - EXPECT_THAT(builder->SetFieldByName("value", BytesValue("foo")), + EXPECT_THAT(builder->SetFieldByName("value", BytesValue::WrapUnsafe("foo")), IsOkAndHolds(Eq(std::nullopt))); - EXPECT_THAT(builder->SetFieldByName("does_not_exist", BytesValue("foo")), - IsOkAndHolds(Optional( - ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); + EXPECT_THAT( + builder->SetFieldByName("does_not_exist", BytesValue::WrapUnsafe("foo")), + IsOkAndHolds( + Optional(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); - EXPECT_THAT(builder->SetFieldByNumber(1, BytesValue("foo")), + EXPECT_THAT(builder->SetFieldByNumber(1, BytesValue::WrapUnsafe("foo")), IsOkAndHolds(Eq(std::nullopt))); - EXPECT_THAT(builder->SetFieldByNumber(2, BytesValue("foo")), + EXPECT_THAT(builder->SetFieldByNumber(2, BytesValue::WrapUnsafe("foo")), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); EXPECT_THAT(builder->SetFieldByNumber(1, BoolValue(true)), @@ -549,10 +552,11 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { arena(), internal::GetTestingDescriptorPool(), internal::GetTestingMessageFactory(), "google.protobuf.Any"); ASSERT_THAT(builder, NotNull()); - EXPECT_THAT(builder->SetFieldByName( - "type_url", - StringValue("type.googleapis.com/google.protobuf.BoolValue")), - IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT( + builder->SetFieldByName( + "type_url", StringValue::WrapUnsafe( + "type.googleapis.com/google.protobuf.BoolValue")), + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByName("does_not_exist", IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); @@ -564,10 +568,10 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { EXPECT_THAT(builder->SetFieldByName("value", BoolValue(true)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); - EXPECT_THAT( - builder->SetFieldByNumber( - 1, StringValue("type.googleapis.com/google.protobuf.BoolValue")), - IsOkAndHolds(Eq(std::nullopt))); + EXPECT_THAT(builder->SetFieldByNumber( + 1, StringValue::WrapUnsafe( + "type.googleapis.com/google.protobuf.BoolValue")), + IsOkAndHolds(Eq(std::nullopt))); EXPECT_THAT(builder->SetFieldByNumber(3, IntValue(1)), IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kNotFound))))); diff --git a/common/value.cc b/common/value.cc index 3feadeb17..1656d4857 100644 --- a/common/value.cc +++ b/common/value.cc @@ -54,6 +54,7 @@ #include "internal/number.h" #include "internal/protobuf_runtime_version.h" #include "internal/status_macros.h" +#include "internal/utf8.h" #include "internal/well_known_types.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -443,10 +444,14 @@ void StringMapFieldKeyAccessor(const google::protobuf::MapKey& key, ABSL_DCHECK(result != nullptr); #if CEL_INTERNAL_PROTOBUF_OSS_VERSION_PREREQ(5, 30, 0) - *result = StringValue(Borrower::Arena(MessageArenaOr(message, arena)), - key.GetStringValue()); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + *result = StringValue::Wrap(key.GetStringValue(), message_arena); + } else { + *result = StringValue::From(key.GetStringValue(), arena); + } #else - *result = StringValue(arena, key.GetStringValue()); + *result = StringValue::From(key.GetStringValue(), arena); #endif } @@ -625,9 +630,9 @@ void StringMapFieldValueAccessor( ABSL_DCHECK_EQ(field->type(), google::protobuf::FieldDescriptor::TYPE_STRING); if (message->GetArena() == nullptr) { - *result = StringValue(arena, value.GetStringValue()); + *result = StringValue::From(value.GetStringValue(), arena); } else { - *result = StringValue(Borrower::Arena(arena), value.GetStringValue()); + *result = StringValue::Wrap(value.GetStringValue(), message->GetArena()); } } @@ -668,9 +673,9 @@ void BytesMapFieldValueAccessor( ABSL_DCHECK_EQ(field->type(), google::protobuf::FieldDescriptor::TYPE_BYTES); if (message->GetArena() == nullptr) { - *result = BytesValue(arena, value.GetStringValue()); + *result = BytesValue::From(value.GetStringValue(), arena); } else { - *result = BytesValue(Borrower::Arena(arena), value.GetStringValue()); + *result = BytesValue::Wrap(value.GetStringValue(), message->GetArena()); } } @@ -961,16 +966,18 @@ void StringRepeatedFieldAccessor( [&](absl::string_view string) { if (string.data() == scratch.data() && string.size() == scratch.size()) { - *result = StringValue(arena, std::move(scratch)); + *result = StringValue::From(std::move(scratch), arena); } else { if (message->GetArena() == nullptr) { - *result = StringValue(arena, string); + *result = StringValue::From(string, arena); } else { - *result = StringValue(Borrower::Arena(arena), string); + *result = StringValue::Wrap(string, message->GetArena()); } } }, - [&](absl::Cord&& cord) { *result = StringValue(std::move(cord)); }), + [&](absl::Cord&& cord) { + *result = StringValue::From(std::move(cord), arena); + }), well_known_types::AsVariant(well_known_types::GetRepeatedStringField( *message, field, index, scratch))); } @@ -1028,16 +1035,18 @@ void BytesRepeatedFieldAccessor( [&](absl::string_view string) { if (string.data() == scratch.data() && string.size() == scratch.size()) { - *result = BytesValue(arena, std::move(scratch)); + *result = BytesValue::From(std::move(scratch), arena); } else { if (message->GetArena() == nullptr) { - *result = BytesValue(arena, string); + *result = BytesValue::From(string, arena); } else { - *result = BytesValue(Borrower::Arena(arena), string); + *result = BytesValue::Wrap(string, message->GetArena()); } } }, - [&](absl::Cord&& cord) { *result = BytesValue(std::move(cord)); }), + [&](absl::Cord&& cord) { + *result = BytesValue::From(std::move(cord), arena); + }), well_known_types::AsVariant(well_known_types::GetRepeatedBytesField( *message, field, index, scratch))); } @@ -1187,15 +1196,16 @@ struct OwningWellKnownTypesValueVisitor { } if (scratch->data() == string.data() && scratch->size() == string.size()) { - return BytesValue(arena, std::move(*scratch)); + return BytesValue::From(std::move(*scratch), + arena); } - return BytesValue(arena, string); + return BytesValue::From(string, arena); }, [&](absl::Cord&& cord) -> BytesValue { if (cord.empty()) { return BytesValue(); } - return BytesValue(arena, cord); + return BytesValue::From(cord, arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1208,15 +1218,16 @@ struct OwningWellKnownTypesValueVisitor { } if (scratch->data() == string.data() && scratch->size() == string.size()) { - return StringValue(arena, std::move(*scratch)); + return StringValue::From(std::move(*scratch), + arena); } - return StringValue(arena, string); + return StringValue::From(string, arena); }, [&](absl::Cord&& cord) -> StringValue { if (cord.empty()) { return StringValue(); } - return StringValue(arena, cord); + return StringValue::From(cord, arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1285,14 +1296,17 @@ struct BorrowingWellKnownTypesValueVisitor { [&](absl::string_view string) -> BytesValue { if (string.data() == scratch->data() && string.size() == scratch->size()) { - return BytesValue(arena, std::move(*scratch)); + return BytesValue::From(std::move(*scratch), arena); } else { - return BytesValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return BytesValue::Wrap(string, message_arena); + } + return BytesValue::From(string, arena); } }, [&](absl::Cord&& cord) -> BytesValue { - return BytesValue(std::move(cord)); + return BytesValue::From(std::move(cord), arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1303,14 +1317,17 @@ struct BorrowingWellKnownTypesValueVisitor { [&](absl::string_view string) -> StringValue { if (string.data() == scratch->data() && string.size() == scratch->size()) { - return StringValue(arena, std::move(*scratch)); + return StringValue::From(std::move(*scratch), arena); } else { - return StringValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); } }, [&](absl::Cord&& cord) -> StringValue { - return StringValue(std::move(cord)); + return StringValue::From(std::move(cord), arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1585,17 +1602,20 @@ Value WrapFieldImpl( [&](absl::string_view string) -> StringValue { if (string.data() == scratch.data() && string.size() == scratch.size()) { - return StringValue(arena, std::move(scratch)); + return StringValue::From(std::move(scratch), arena); } if constexpr (Unsafe::value) { return StringValue::WrapUnsafe(string); } else { - return StringValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); } }, [&](absl::Cord&& cord) -> StringValue { - return StringValue(std::move(cord)); + return StringValue::From(std::move(cord), arena); }), well_known_types::AsVariant( well_known_types::GetStringField(*message, field, scratch))); @@ -1624,17 +1644,20 @@ Value WrapFieldImpl( [&](absl::string_view string) -> BytesValue { if (string.data() == scratch.data() && string.size() == scratch.size()) { - return BytesValue(arena, std::move(scratch)); + return BytesValue::From(std::move(scratch), arena); } if constexpr (Unsafe::value) { return BytesValue::WrapUnsafe(string); } else { - return BytesValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return BytesValue::Wrap(string, message_arena); + } + return BytesValue::From(string, arena); } }, [&](absl::Cord&& cord) -> BytesValue { - return BytesValue(std::move(cord)); + return BytesValue::From(std::move(cord), arena); }), well_known_types::AsVariant( well_known_types::GetBytesField(*message, field, scratch))); @@ -1726,17 +1749,20 @@ Value WrapRepeatedFieldImpl( [&](absl::string_view string) -> StringValue { if (string.data() == scratch.data() && string.size() == scratch.size()) { - return StringValue(arena, std::move(scratch)); + return StringValue::From(std::move(scratch), arena); } if constexpr (Unsafe::value) { return StringValue::WrapUnsafe(string); } else { - return StringValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); } }, [&](absl::Cord&& cord) -> StringValue { - return StringValue(std::move(cord)); + return StringValue::From(std::move(cord), arena); }), well_known_types::AsVariant(well_known_types::GetRepeatedStringField( reflection, *message, field, index, scratch))); @@ -1760,17 +1786,20 @@ Value WrapRepeatedFieldImpl( [&](absl::string_view string) -> BytesValue { if (string.data() == scratch.data() && string.size() == scratch.size()) { - return BytesValue(arena, std::move(scratch)); + return BytesValue::From(std::move(scratch), arena); } if constexpr (Unsafe::value) { return BytesValue::WrapUnsafe(string); } else { - return BytesValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return BytesValue::Wrap(string, message_arena); + } + return BytesValue::From(string, arena); } }, [&](absl::Cord&& cord) -> BytesValue { - return BytesValue(std::move(cord)); + return BytesValue::From(std::move(cord), arena); }), well_known_types::AsVariant(well_known_types::GetRepeatedBytesField( reflection, *message, field, index, scratch))); @@ -1838,8 +1867,7 @@ Value WrapMapFieldValueImpl( if constexpr (Unsafe::value) { return StringValue::WrapUnsafe(value.GetStringValue()); } else { - return StringValue(Borrower::Arena(MessageArenaOr(message, arena)), - value.GetStringValue()); + return StringValue::From(value.GetStringValue(), arena); } case google::protobuf::FieldDescriptor::TYPE_GROUP: ABSL_FALLTHROUGH_INTENDED; @@ -1855,8 +1883,7 @@ Value WrapMapFieldValueImpl( if constexpr (Unsafe::value) { return BytesValue::WrapUnsafe(value.GetStringValue()); } else { - return BytesValue(Borrower::Arena(MessageArenaOr(message, arena)), - value.GetStringValue()); + return BytesValue::From(value.GetStringValue(), arena); } case google::protobuf::FieldDescriptor::TYPE_FIXED32: ABSL_FALLTHROUGH_INTENDED; @@ -1943,11 +1970,12 @@ StringValue Value::WrapMapFieldKeyString( ABSL_DCHECK_EQ(key.type(), google::protobuf::FieldDescriptor::CPPTYPE_STRING); #if CEL_INTERNAL_PROTOBUF_OSS_VERSION_PREREQ(5, 30, 0) - return StringValue(Borrower::Arena(MessageArenaOr(message, arena)), - key.GetStringValue()); -#else - return StringValue(arena, key.GetStringValue()); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(key.GetStringValue(), message_arena); + } #endif + return StringValue::From(key.GetStringValue(), arena); } Value Value::WrapMapFieldValue( @@ -2831,4 +2859,14 @@ absl::StatusOr ValueIterator::Next1( return false; } +StringValue::StringValue(const BytesValue& other) : StringValue(other.value_) { + ABSL_DCHECK(value_.Visit(absl::Overload( + [](absl::string_view string) -> bool { + return internal::Utf8IsValid(string); + }, + [](const absl::Cord& cord) -> bool { + return internal::Utf8IsValid(cord); + }))); +} + } // namespace cel diff --git a/common/value.h b/common/value.h index c13dad36e..881f8c925 100644 --- a/common/value.h +++ b/common/value.h @@ -2944,6 +2944,9 @@ absl::StatusOr RepeatedFieldAccessorFor( } // namespace common_internal +inline BytesValue::BytesValue(const StringValue& other) + : BytesValue(other.value_) {} + } // namespace cel #pragma pop_macro("GetMessage") diff --git a/common/value_testing_test.cc b/common/value_testing_test.cc index d7a7a4c07..425ce92cc 100644 --- a/common/value_testing_test.cc +++ b/common/value_testing_test.cc @@ -25,6 +25,7 @@ namespace cel::test { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::testing::_; using ::testing::ElementsAre; @@ -124,11 +125,12 @@ TEST(TimestampValueIs, NonMatchMessage) { } TEST(StringValueIs, Match) { - EXPECT_THAT(StringValue("hello!"), StringValueIs("hello!")); + EXPECT_THAT(StringValue::WrapUnsafe("hello!"), StringValueIs("hello!")); } TEST(StringValueIs, NoMatch) { - EXPECT_THAT(StringValue("hello!"), Not(StringValueIs("goodbye!"))); + EXPECT_THAT(StringValue::WrapUnsafe("hello!"), + Not(StringValueIs("goodbye!"))); EXPECT_THAT(IntValue(2), Not(StringValueIs("goodbye!"))); } @@ -139,11 +141,11 @@ TEST(StringValueIs, NonMatchMessage) { } TEST(BytesValueIs, Match) { - EXPECT_THAT(BytesValue("hello!"), BytesValueIs("hello!")); + EXPECT_THAT(BytesValue::WrapUnsafe("hello!"), BytesValueIs("hello!")); } TEST(BytesValueIs, NoMatch) { - EXPECT_THAT(BytesValue("hello!"), Not(BytesValueIs("goodbye!"))); + EXPECT_THAT(BytesValue::WrapUnsafe("hello!"), Not(BytesValueIs("goodbye!"))); EXPECT_THAT(IntValue(2), Not(BytesValueIs("goodbye!"))); } @@ -265,8 +267,10 @@ TEST_F(ValueMatcherTest, MapMatcherBasic) { TEST_F(ValueMatcherTest, MapMatcherMatchesElements) { auto builder = NewMapValueBuilder(arena()); - ASSERT_OK(builder->Put(IntValue(42), StringValue("answer"))); - ASSERT_OK(builder->Put(IntValue(1337), StringValue("leet"))); + ASSERT_THAT(builder->Put(IntValue(42), StringValue::WrapUnsafe("answer")), + IsOk()); + ASSERT_THAT(builder->Put(IntValue(1337), StringValue::WrapUnsafe("leet")), + IsOk()); EXPECT_THAT( std::move(*builder).Build(), MapValueIs(MapValueElements( diff --git a/common/values/bytes_value.h b/common/values/bytes_value.h index c18381a6a..461e5d8c6 100644 --- a/common/values/bytes_value.h +++ b/common/values/bytes_value.h @@ -26,15 +26,11 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" -#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/arena.h" #include "common/internal/byte_string.h" -#include "common/memory.h" #include "common/type.h" #include "common/value_kind.h" #include "common/values/values.h" @@ -62,75 +58,61 @@ class BytesValue final : private common_internal::ValueMixin { static BytesValue From(const char* absl_nullable value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::From(value, arena)); + } static BytesValue From(absl::string_view value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static BytesValue From(const absl::Cord& value); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::From(value, arena)); + } + static BytesValue From(const absl::Cord& value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::From(value, arena)); + } static BytesValue From(std::string&& value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue( + common_internal::ByteString::From(std::move(value), arena)); + } static BytesValue Wrap(absl::string_view value, google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static BytesValue Wrap(absl::string_view value) = delete; - static BytesValue Wrap(const absl::Cord& value); - static BytesValue Wrap(std::string&& value) = delete; - static BytesValue Wrap(std::string&& value, - google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) = delete; + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::Wrap(value, arena)); + } + static BytesValue Wrap( + const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return BytesValue(common_internal::ByteString::Wrap(value, arena)); + } + static BytesValue Wrap(std::nullptr_t, google::protobuf::Arena*) = delete; + static BytesValue Wrap(std::string&& value, google::protobuf::Arena*) = delete; // Returns a BytesValue that aliases the provided string. Caller must ensure // the provided string outlives the use of the returned BytesValue. - static BytesValue WrapUnsafe(absl::string_view value); + static BytesValue WrapUnsafe(absl::string_view value) { + return BytesValue(common_internal::ByteString::WrapUnsafe(value)); + } + static BytesValue WrapUnsafe(const absl::Cord* absl_nonnull value) { + return BytesValue(common_internal::ByteString::WrapUnsafe(value)); + } + static BytesValue WrapUnsafe(std::nullptr_t) = delete; static BytesValue Concat(const BytesValue& lhs, const BytesValue& rhs, google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND); - ABSL_DEPRECATED("Use From") - explicit BytesValue(const char* absl_nullable value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit BytesValue(absl::string_view value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit BytesValue(const absl::Cord& value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit BytesValue(std::string&& value) : value_(std::move(value)) {} - - ABSL_DEPRECATED("Use From") - BytesValue(Allocator<> allocator, const char* absl_nullable value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - BytesValue(Allocator<> allocator, absl::string_view value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - BytesValue(Allocator<> allocator, const absl::Cord& value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - BytesValue(Allocator<> allocator, std::string&& value) - : value_(allocator, std::move(value)) {} - - ABSL_DEPRECATED("Use Wrap") - BytesValue(Borrower borrower, absl::string_view value) - : value_(borrower, value) {} - - ABSL_DEPRECATED("Use Wrap") - BytesValue(Borrower borrower, const absl::Cord& value) - : value_(borrower, value) {} - BytesValue() = default; BytesValue(const BytesValue&) = default; BytesValue(BytesValue&&) = default; BytesValue& operator=(const BytesValue&) = default; BytesValue& operator=(BytesValue&&) = default; + explicit BytesValue(const StringValue& other); + constexpr ValueKind kind() const { return kKind; } absl::string_view GetTypeName() const { return BytesType::kName; } @@ -185,9 +167,9 @@ class BytesValue final : private common_internal::ValueMixin { return value_.Visit(std::forward(visitor)); } - void swap(BytesValue& other) noexcept { + friend void swap(BytesValue& lhs, BytesValue& rhs) noexcept { using std::swap; - swap(value_, other.value_); + swap(lhs.value_, rhs.value_); } size_t Size() const; @@ -238,12 +220,12 @@ class BytesValue final : private common_internal::ValueMixin { } private: + friend class StringValue; friend class common_internal::ValueMixin; friend class BytesValueInputStream; friend class BytesValueOutputStream; friend absl::string_view common_internal::LegacyBytesValue( const BytesValue& value, bool stable, google::protobuf::Arena* absl_nonnull arena); - friend struct ArenaTraits; explicit BytesValue(common_internal::ByteString value) noexcept : value_(std::move(value)) {} @@ -251,8 +233,6 @@ class BytesValue final : private common_internal::ValueMixin { common_internal::ByteString value_; }; -inline void swap(BytesValue& lhs, BytesValue& rhs) noexcept { lhs.swap(rhs); } - inline std::ostream& operator<<(std::ostream& out, const BytesValue& value) { return out << value.DebugString(); } @@ -273,48 +253,6 @@ inline bool operator!=(absl::string_view lhs, const BytesValue& rhs) { return rhs != lhs; } -inline BytesValue BytesValue::From(const char* absl_nullable value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return From(absl::NullSafeStringView(value), arena); -} - -inline BytesValue BytesValue::From(absl::string_view value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return BytesValue(arena, value); -} - -inline BytesValue BytesValue::From(const absl::Cord& value) { - return BytesValue(value); -} - -inline BytesValue BytesValue::From(std::string&& value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return BytesValue(arena, std::move(value)); -} - -inline BytesValue BytesValue::Wrap(absl::string_view value, - google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return BytesValue(Borrower::Arena(arena), value); -} - -inline BytesValue BytesValue::WrapUnsafe(absl::string_view value) { - return BytesValue(common_internal::ByteString::FromExternal(value)); -} - -inline BytesValue BytesValue::Wrap(const absl::Cord& value) { - return BytesValue(value); -} - namespace common_internal { inline absl::string_view LegacyBytesValue(const BytesValue& value, bool stable, @@ -324,15 +262,6 @@ inline absl::string_view LegacyBytesValue(const BytesValue& value, bool stable, } // namespace common_internal -template <> -struct ArenaTraits { - using constructible = std::true_type; - - static bool trivially_destructible(const BytesValue& value) { - return ArenaTraits<>::trivially_destructible(value.value_); - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_BYTES_VALUE_H_ diff --git a/common/values/bytes_value_input_stream.h b/common/values/bytes_value_input_stream.h index c4224f30d..35050d2de 100644 --- a/common/values/bytes_value_input_stream.h +++ b/common/values/bytes_value_input_stream.h @@ -49,27 +49,31 @@ class BytesValueInputStream final : public google::protobuf::io::ZeroCopyInputSt bool Next(const void** data, int* size) override { return absl::visit( [&data, &size](auto& alternative) -> bool { - return alternative.Next(data, size); + return alternative.stream.Next(data, size); }, AsVariant()); } void BackUp(int count) override { absl::visit( - [&count](auto& alternative) -> void { alternative.BackUp(count); }, + [&count](auto& alternative) -> void { + alternative.stream.BackUp(count); + }, AsVariant()); } bool Skip(int count) override { return absl::visit( - [&count](auto& alternative) -> bool { return alternative.Skip(count); }, + [&count](auto& alternative) -> bool { + return alternative.stream.Skip(count); + }, AsVariant()); } int64_t ByteCount() const override { return absl::visit( [](const auto& alternative) -> int64_t { - return alternative.ByteCount(); + return alternative.stream.ByteCount(); }, AsVariant()); } @@ -77,14 +81,25 @@ class BytesValueInputStream final : public google::protobuf::io::ZeroCopyInputSt bool ReadCord(absl::Cord* cord, int count) override { return absl::visit( [&cord, &count](auto& alternative) -> bool { - return alternative.ReadCord(cord, count); + return alternative.stream.ReadCord(cord, count); }, AsVariant()); } private: - using Variant = - absl::variant; + struct ArrayStream { + ArrayStream(const char* data, int size) : stream(data, size) {} + + google::protobuf::io::ArrayInputStream stream; + }; + struct CordStream { + explicit CordStream(const absl::Cord& cord) + : cord(cord), stream(&this->cord) {} + + absl::Cord cord; + google::protobuf::io::CordInputStream stream; + }; + using Variant = absl::variant; void Construct(const BytesValue* absl_nonnull value) { ABSL_DCHECK(value != nullptr); @@ -97,7 +112,7 @@ class BytesValueInputStream final : public google::protobuf::io::ZeroCopyInputSt Construct(value->value_.GetMedium()); break; case common_internal::ByteStringKind::kLarge: - Construct(&value->value_.GetLarge()); + Construct(value->value_.GetLarge()); break; } } @@ -106,13 +121,13 @@ class BytesValueInputStream final : public google::protobuf::io::ZeroCopyInputSt ABSL_DCHECK_LE(value.size(), static_cast(std::numeric_limits::max())); ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value.data(), + Variant(absl::in_place_type, value.data(), static_cast(value.size())); } - void Construct(const absl::Cord* absl_nonnull value) { + void Construct(const absl::Cord& value) { ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value); + Variant(absl::in_place_type, value); } void Destruct() { AsVariant().~variant(); } diff --git a/common/values/bytes_value_output_stream.h b/common/values/bytes_value_output_stream.h index 0773e40e7..b23d1bd3d 100644 --- a/common/values/bytes_value_output_stream.h +++ b/common/values/bytes_value_output_stream.h @@ -26,6 +26,7 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/functional/overload.h" +#include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/variant.h" @@ -40,13 +41,7 @@ namespace cel { class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutputStream { public: - explicit BytesValueOutputStream(const BytesValue& value) - : BytesValueOutputStream(value, /*arena=*/nullptr) {} - - BytesValueOutputStream(const BytesValue& value, - google::protobuf::Arena* absl_nullable arena) { - Construct(value, arena); - } + explicit BytesValueOutputStream(const BytesValue& value) { Construct(value); } bool Next(void** data, int* size) override { return absl::visit(absl::Overload( @@ -54,7 +49,7 @@ class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutput return string.stream.Next(data, size); }, [&data, &size](Cord& cord) -> bool { - return cord.Next(data, size); + return cord.stream.Next(data, size); }), AsVariant()); } @@ -63,18 +58,19 @@ class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutput absl::visit( absl::Overload( [&count](String& string) -> void { string.stream.BackUp(count); }, - [&count](Cord& cord) -> void { cord.BackUp(count); }), + [&count](Cord& cord) -> void { cord.stream.BackUp(count); }), AsVariant()); } int64_t ByteCount() const override { - return absl::visit( - absl::Overload( - [](const String& string) -> int64_t { - return string.stream.ByteCount(); - }, - [](const Cord& cord) -> int64_t { return cord.ByteCount(); }), - AsVariant()); + return absl::visit(absl::Overload( + [](const String& string) -> int64_t { + return string.stream.ByteCount(); + }, + [](const Cord& cord) -> int64_t { + return cord.stream.ByteCount(); + }), + AsVariant()); } bool WriteAliasedRaw(const void* data, int size) override { @@ -83,19 +79,20 @@ class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutput return string.stream.WriteAliasedRaw(data, size); }, [&data, &size](Cord& cord) -> bool { - return cord.WriteAliasedRaw(data, size); + return cord.stream.WriteAliasedRaw(data, size); }), AsVariant()); } bool AllowsAliasing() const override { - return absl::visit( - absl::Overload( - [](const String& string) -> bool { - return string.stream.AllowsAliasing(); - }, - [](const Cord& cord) -> bool { return cord.AllowsAliasing(); }), - AsVariant()); + return absl::visit(absl::Overload( + [](const String& string) -> bool { + return string.stream.AllowsAliasing(); + }, + [](const Cord& cord) -> bool { + return cord.stream.AllowsAliasing(); + }), + AsVariant()); } bool WriteCord(const absl::Cord& out) override { @@ -104,43 +101,47 @@ class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutput [&out](String& string) -> bool { return string.stream.WriteCord(out); }, - [&out](Cord& cord) -> bool { return cord.WriteCord(out); }), + [&out](Cord& cord) -> bool { return cord.stream.WriteCord(out); }), AsVariant()); } - BytesValue Consume() && { - return absl::visit(absl::Overload( - [](String& string) -> BytesValue { - return BytesValue(string.arena, - std::move(string.target)); - }, - [](Cord& cord) -> BytesValue { - return BytesValue(cord.Consume()); - }), - AsVariant()); + BytesValue Consume(google::protobuf::Arena* absl_nonnull arena) && { + ABSL_DCHECK(arena != nullptr); + return absl::visit( + absl::Overload( + [arena](String& string) -> BytesValue { + return BytesValue::From(std::move(string.target), arena); + }, + [arena](Cord& cord) -> BytesValue { + return BytesValue::From(cord.stream.Consume(), arena); + }), + AsVariant()); } private: struct String final { - String(absl::string_view target, google::protobuf::Arena* absl_nullable arena) - : target(target), stream(&this->target), arena(arena) {} + explicit String(absl::string_view target) + : target(target), stream(&this->target) {} std::string target; google::protobuf::io::StringOutputStream stream; - google::protobuf::Arena* absl_nullable arena; }; - using Cord = google::protobuf::io::CordOutputStream; + struct Cord final { + explicit Cord(const absl::Cord& cord) : stream(cord) {} + + google::protobuf::io::CordOutputStream stream; + }; using Variant = absl::variant; - void Construct(const BytesValue& value, google::protobuf::Arena* absl_nullable arena) { + void Construct(const BytesValue& value) { switch (value.value_.GetKind()) { case common_internal::ByteStringKind::kSmall: - Construct(value.value_.GetSmall(), arena); + Construct(value.value_.GetSmall()); break; case common_internal::ByteStringKind::kMedium: - Construct(value.value_.GetMedium(), arena); + Construct(value.value_.GetMedium()); break; case common_internal::ByteStringKind::kLarge: Construct(value.value_.GetLarge()); @@ -148,9 +149,9 @@ class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutput } } - void Construct(absl::string_view value, google::protobuf::Arena* absl_nullable arena) { + void Construct(absl::string_view value) { ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value, arena); + Variant(absl::in_place_type, value); } void Construct(const absl::Cord& value) { diff --git a/common/values/bytes_value_test.cc b/common/values/bytes_value_test.cc index 367a8ca16..e4e7ad665 100644 --- a/common/values/bytes_value_test.cc +++ b/common/values/bytes_value_test.cc @@ -20,7 +20,6 @@ #include "absl/status/status_matchers.h" #include "absl/strings/cord.h" #include "absl/strings/cord_test_helpers.h" -#include "absl/types/optional.h" #include "common/native_type.h" #include "common/value.h" #include "common/value_testing.h" @@ -38,130 +37,139 @@ using ::testing::Optional; using BytesValueTest = common_internal::ValueTest<>; TEST_F(BytesValueTest, Kind) { - EXPECT_EQ(BytesValue("foo").kind(), BytesValue::kKind); - EXPECT_EQ(Value(BytesValue(absl::Cord("foo"))).kind(), BytesValue::kKind); + EXPECT_EQ(BytesValue::WrapUnsafe("foo").kind(), BytesValue::kKind); + EXPECT_EQ(Value(BytesValue::From(absl::Cord("foo"), arena())).kind(), + BytesValue::kKind); } TEST_F(BytesValueTest, DebugString) { { std::ostringstream out; - out << BytesValue("foo"); + out << BytesValue::WrapUnsafe("foo"); EXPECT_EQ(out.str(), "b\"foo\""); } { std::ostringstream out; - out << BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})); + out << BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()); EXPECT_EQ(out.str(), "b\"foo\""); } { std::ostringstream out; - out << Value(BytesValue(absl::Cord("foo"))); + out << Value(BytesValue::From(absl::Cord("foo"), arena())); EXPECT_EQ(out.str(), "b\"foo\""); } } TEST_F(BytesValueTest, ConvertToJson) { auto* message = NewArenaValueMessage(); - EXPECT_THAT(BytesValue("foo").ConvertToJson(descriptor_pool(), - message_factory(), message), + EXPECT_THAT(BytesValue::WrapUnsafe("foo").ConvertToJson( + descriptor_pool(), message_factory(), message), IsOk()); EXPECT_THAT(*message, EqualsValueTextProto(R"pb(string_value: "Zm9v")pb")); } TEST_F(BytesValueTest, NativeValue) { std::string scratch; - EXPECT_EQ(BytesValue("foo").NativeString(), "foo"); - EXPECT_EQ(BytesValue("foo").NativeString(scratch), "foo"); - EXPECT_EQ(BytesValue("foo").NativeCord(), "foo"); + EXPECT_EQ(BytesValue::WrapUnsafe("foo").NativeString(), "foo"); + EXPECT_EQ(BytesValue::WrapUnsafe("foo").NativeString(scratch), "foo"); + EXPECT_EQ(BytesValue::WrapUnsafe("foo").NativeCord(), "foo"); } TEST_F(BytesValueTest, TryFlat) { - EXPECT_THAT(BytesValue("foo").TryFlat(), Optional(Eq("foo"))); + EXPECT_THAT(BytesValue::WrapUnsafe("foo").TryFlat(), Optional(Eq("foo"))); EXPECT_THAT( - BytesValue(absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"})) + BytesValue::From( + absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"}), arena()) .TryFlat(), Eq(std::nullopt)); } TEST_F(BytesValueTest, ToString) { - EXPECT_EQ(BytesValue("foo").ToString(), "foo"); - EXPECT_EQ(BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).ToString(), + EXPECT_EQ(BytesValue::WrapUnsafe("foo").ToString(), "foo"); + EXPECT_EQ(BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .ToString(), "foo"); } TEST_F(BytesValueTest, CopyToString) { std::string out; - BytesValue("foo").CopyToString(&out); + BytesValue::WrapUnsafe("foo").CopyToString(&out); EXPECT_EQ(out, "foo"); - BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).CopyToString(&out); + BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .CopyToString(&out); EXPECT_EQ(out, "foo"); } TEST_F(BytesValueTest, AppendToString) { std::string out; - BytesValue("foo").AppendToString(&out); + BytesValue::WrapUnsafe("foo").AppendToString(&out); EXPECT_EQ(out, "foo"); - BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).AppendToString(&out); + BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .AppendToString(&out); EXPECT_EQ(out, "foofoo"); } TEST_F(BytesValueTest, ToCord) { - EXPECT_EQ(BytesValue("foo").ToCord(), "foo"); - EXPECT_EQ(BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).ToCord(), + EXPECT_EQ(BytesValue::WrapUnsafe("foo").ToCord(), "foo"); + EXPECT_EQ(BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .ToCord(), "foo"); } TEST_F(BytesValueTest, CopyToCord) { absl::Cord out; - BytesValue("foo").CopyToCord(&out); + BytesValue::WrapUnsafe("foo").CopyToCord(&out); EXPECT_EQ(out, "foo"); - BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).CopyToCord(&out); + BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .CopyToCord(&out); EXPECT_EQ(out, "foo"); } TEST_F(BytesValueTest, AppendToCord) { absl::Cord out; - BytesValue("foo").AppendToCord(&out); + BytesValue::WrapUnsafe("foo").AppendToCord(&out); EXPECT_EQ(out, "foo"); - BytesValue(absl::MakeFragmentedCord({"f", "o", "o"})).AppendToCord(&out); + BytesValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .AppendToCord(&out); EXPECT_EQ(out, "foofoo"); } TEST_F(BytesValueTest, NativeTypeId) { - EXPECT_EQ(NativeTypeId::Of(BytesValue("foo")), - NativeTypeId::For()); - EXPECT_EQ(NativeTypeId::Of(Value(BytesValue(absl::Cord("foo")))), + EXPECT_EQ(NativeTypeId::Of(BytesValue::WrapUnsafe("foo")), NativeTypeId::For()); + EXPECT_EQ( + NativeTypeId::Of(Value(BytesValue::From(absl::Cord("foo"), arena()))), + NativeTypeId::For()); } TEST_F(BytesValueTest, StringViewEquality) { // NOLINTBEGIN(readability/check) - EXPECT_TRUE(BytesValue("foo") == "foo"); - EXPECT_FALSE(BytesValue("foo") == "bar"); + EXPECT_TRUE(BytesValue::WrapUnsafe("foo") == "foo"); + EXPECT_FALSE(BytesValue::WrapUnsafe("foo") == "bar"); - EXPECT_TRUE("foo" == BytesValue("foo")); - EXPECT_FALSE("bar" == BytesValue("foo")); + EXPECT_TRUE("foo" == BytesValue::WrapUnsafe("foo")); + EXPECT_FALSE("bar" == BytesValue::WrapUnsafe("foo")); // NOLINTEND(readability/check) } TEST_F(BytesValueTest, StringViewInequality) { // NOLINTBEGIN(readability/check) - EXPECT_FALSE(BytesValue("foo") != "foo"); - EXPECT_TRUE(BytesValue("foo") != "bar"); + EXPECT_FALSE(BytesValue::WrapUnsafe("foo") != "foo"); + EXPECT_TRUE(BytesValue::WrapUnsafe("foo") != "bar"); - EXPECT_FALSE("foo" != BytesValue("foo")); - EXPECT_TRUE("bar" != BytesValue("foo")); + EXPECT_FALSE("foo" != BytesValue::WrapUnsafe("foo")); + EXPECT_TRUE("bar" != BytesValue::WrapUnsafe("foo")); // NOLINTEND(readability/check) } TEST_F(BytesValueTest, Comparison) { - EXPECT_LT(BytesValue("bar"), BytesValue("foo")); - EXPECT_FALSE(BytesValue("foo") < BytesValue("foo")); - EXPECT_FALSE(BytesValue("foo") < BytesValue("bar")); + EXPECT_LT(BytesValue::WrapUnsafe("bar"), BytesValue::WrapUnsafe("foo")); + EXPECT_FALSE(BytesValue::WrapUnsafe("foo") < BytesValue::WrapUnsafe("foo")); + EXPECT_FALSE(BytesValue::WrapUnsafe("foo") < BytesValue::WrapUnsafe("bar")); } TEST_F(BytesValueTest, StringInputStream) { - BytesValue value = BytesValue("foo"); + BytesValue value = BytesValue::WrapUnsafe("foo"); BytesValueInputStream stream(&value); const void* data; int size; @@ -177,7 +185,7 @@ TEST_F(BytesValueTest, StringInputStream) { } TEST_F(BytesValueTest, CordInputStream) { - BytesValue value = BytesValue(absl::Cord("foo")); + BytesValue value = BytesValue::From(absl::Cord("foo"), arena()); BytesValueInputStream stream(&value); const void* data; int size; @@ -193,9 +201,9 @@ TEST_F(BytesValueTest, CordInputStream) { } TEST_F(BytesValueTest, ArenaStringOutputStream) { - BytesValue value = BytesValue(""); + BytesValue value = BytesValue(); { - BytesValueOutputStream stream(value, arena()); + BytesValueOutputStream stream(value); EXPECT_THAT(stream.AllowsAliasing(), An()); EXPECT_EQ(stream.ByteCount(), 0); google::protobuf::Value value_proto; @@ -203,17 +211,17 @@ TEST_F(BytesValueTest, ArenaStringOutputStream) { (*struct_proto->mutable_fields())["foo"].set_string_value("bar"); (*struct_proto->mutable_fields())["baz"].set_number_value(3.14159); ASSERT_TRUE(value_proto.SerializePartialToZeroCopyStream(&stream)); - EXPECT_EQ(std::move(stream).Consume(), + EXPECT_EQ(std::move(stream).Consume(arena()), value_proto.SerializePartialAsString()); } { BytesValueOutputStream stream(value); - EXPECT_EQ(std::move(stream).Consume(), ""); + EXPECT_EQ(std::move(stream).Consume(arena()), ""); } } TEST_F(BytesValueTest, StringOutputStream) { - BytesValue value = BytesValue(""); + BytesValue value = BytesValue(); { BytesValueOutputStream stream(value); EXPECT_THAT(stream.AllowsAliasing(), An()); @@ -223,17 +231,18 @@ TEST_F(BytesValueTest, StringOutputStream) { (*struct_proto->mutable_fields())["foo"].set_string_value("bar"); (*struct_proto->mutable_fields())["baz"].set_number_value(3.14159); ASSERT_TRUE(value_proto.SerializePartialToZeroCopyStream(&stream)); - EXPECT_EQ(std::move(stream).Consume(), + EXPECT_EQ(std::move(stream).Consume(arena()), value_proto.SerializePartialAsString()); } { BytesValueOutputStream stream(value); - EXPECT_EQ(std::move(stream).Consume(), ""); + EXPECT_EQ(std::move(stream).Consume(arena()), ""); } } TEST_F(BytesValueTest, CordOutputStream) { - BytesValue value = BytesValue(absl::Cord()); + absl::Cord cord; + BytesValue value = BytesValue::WrapUnsafe(&cord); { BytesValueOutputStream stream(value); EXPECT_THAT(stream.AllowsAliasing(), An()); @@ -243,12 +252,12 @@ TEST_F(BytesValueTest, CordOutputStream) { (*struct_proto->mutable_fields())["foo"].set_string_value("bar"); (*struct_proto->mutable_fields())["baz"].set_number_value(3.14159); ASSERT_TRUE(value_proto.SerializePartialToZeroCopyStream(&stream)); - EXPECT_EQ(std::move(stream).Consume(), + EXPECT_EQ(std::move(stream).Consume(arena()), value_proto.SerializePartialAsString()); } { BytesValueOutputStream stream(value); - EXPECT_EQ(std::move(stream).Consume(), ""); + EXPECT_EQ(std::move(stream).Consume(arena()), ""); } } diff --git a/common/values/custom_list_value.cc b/common/values/custom_list_value.cc index fbba38cfa..ca001b103 100644 --- a/common/values/custom_list_value.cc +++ b/common/values/custom_list_value.cc @@ -486,10 +486,7 @@ CustomListValue CustomListValue::Clone( CustomListValueInterface::Content content = content_.To(); ABSL_DCHECK(content.interface != nullptr); - if (content.arena != arena) { - return content.interface->Clone(arena); - } - return *this; + return content.interface->Clone(arena); } return dispatcher_->clone(dispatcher_, content_, arena); } diff --git a/common/values/custom_map_value.cc b/common/values/custom_map_value.cc index 7efb1de03..99e610efe 100644 --- a/common/values/custom_map_value.cc +++ b/common/values/custom_map_value.cc @@ -589,10 +589,7 @@ CustomMapValue CustomMapValue::Clone(google::protobuf::Arena* absl_nonnull arena CustomMapValueInterface::Content content = content_.To(); ABSL_DCHECK(content.interface != nullptr); - if (content.arena != arena) { - return content.interface->Clone(arena); - } - return *this; + return content.interface->Clone(arena); } return dispatcher_->clone(dispatcher_, content_, arena); } diff --git a/common/values/custom_map_value_test.cc b/common/values/custom_map_value_test.cc index 11c46d4cf..2eb833ac0 100644 --- a/common/values/custom_map_value_test.cc +++ b/common/values/custom_map_value_test.cc @@ -109,8 +109,8 @@ class CustomMapValueInterfaceTest final : public CustomMapValueInterface { ListValue* absl_nonnull result) const override { auto builder = common_internal::NewListValueBuilder(arena); builder->Reserve(2); - CEL_RETURN_IF_ERROR(builder->Add(StringValue("foo"))); - CEL_RETURN_IF_ERROR(builder->Add(StringValue("bar"))); + CEL_RETURN_IF_ERROR(builder->Add(StringValue::WrapUnsafe("foo"))); + CEL_RETURN_IF_ERROR(builder->Add(StringValue::WrapUnsafe("bar"))); *result = std::move(*builder).Build(); return absl::OkStatus(); } @@ -293,8 +293,8 @@ class CustomMapValueTest : public common_internal::ValueTest<> { ListValue* absl_nonnull result) -> absl::Status { auto builder = common_internal::NewListValueBuilder(arena); builder->Reserve(2); - CEL_RETURN_IF_ERROR(builder->Add(StringValue("foo"))); - CEL_RETURN_IF_ERROR(builder->Add(StringValue("bar"))); + CEL_RETURN_IF_ERROR(builder->Add(StringValue::WrapUnsafe("foo"))); + CEL_RETURN_IF_ERROR(builder->Add(StringValue::WrapUnsafe("bar"))); *result = std::move(*builder).Build(); return absl::OkStatus(); }, @@ -456,76 +456,76 @@ TEST_F(CustomMapValueTest, Interface_Size) { TEST_F(CustomMapValueTest, Dispatcher_Get) { CustomMapValue map = MakeDispatcher(); - ASSERT_THAT(map.Get(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Get(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(IntValueIs(1))); ASSERT_THAT( - map.Get(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + map.Get(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); } TEST_F(CustomMapValueTest, Interface_Get) { CustomMapValue map = MakeInterface(); - ASSERT_THAT(map.Get(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Get(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(IntValueIs(1))); ASSERT_THAT( - map.Get(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + map.Get(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); } TEST_F(CustomMapValueTest, Dispatcher_Find) { CustomMapValue map = MakeDispatcher(); - ASSERT_THAT(map.Find(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); - ASSERT_THAT(map.Find(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Optional(IntValueIs(1)))); - ASSERT_THAT(map.Find(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Interface_Find) { CustomMapValue map = MakeInterface(); - ASSERT_THAT(map.Find(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); - ASSERT_THAT(map.Find(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Optional(IntValueIs(1)))); - ASSERT_THAT(map.Find(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(Eq(std::nullopt))); } TEST_F(CustomMapValueTest, Dispatcher_Find_Error) { CustomMapValue map = MakeDispatcher(); Value result; - ASSERT_THAT(map.Find(StringValue("error"), descriptor_pool(), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena(), &result), IsOkAndHolds(false)); EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, "custom error"))); - ASSERT_THAT(map.Get(StringValue("error"), descriptor_pool(), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena(), &result), IsOk()); EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, "custom error"))); - EXPECT_THAT(map.Get(StringValue("error"), descriptor_pool(), + EXPECT_THAT(map.Get(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs( absl::StatusCode::kInvalidArgument, "custom error")))); - EXPECT_THAT(map.Find(StringValue("error"), descriptor_pool(), + EXPECT_THAT(map.Find(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Eq(std::nullopt))); } @@ -533,21 +533,21 @@ TEST_F(CustomMapValueTest, Dispatcher_Find_Error) { TEST_F(CustomMapValueTest, Interface_Find_Error) { CustomMapValue map = MakeInterface(); Value result; - ASSERT_THAT(map.Find(StringValue("error"), descriptor_pool(), + ASSERT_THAT(map.Find(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena(), &result), IsOkAndHolds(false)); EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, "custom error"))); - ASSERT_THAT(map.Get(StringValue("error"), descriptor_pool(), + ASSERT_THAT(map.Get(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena(), &result), IsOk()); EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, "custom error"))); - EXPECT_THAT(map.Get(StringValue("error"), descriptor_pool(), + EXPECT_THAT(map.Get(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs( absl::StatusCode::kInvalidArgument, "custom error")))); - EXPECT_THAT(map.Find(StringValue("error"), descriptor_pool(), + EXPECT_THAT(map.Find(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Eq(std::nullopt))); } @@ -654,39 +654,39 @@ TEST_F(CustomMapValueTest, Interface_Find_SpecialKeys) { TEST_F(CustomMapValueTest, Dispatcher_Has) { CustomMapValue map = MakeDispatcher(); - ASSERT_THAT(map.Has(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Has(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Has(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); } TEST_F(CustomMapValueTest, Interface_Has) { CustomMapValue map = MakeInterface(); - ASSERT_THAT(map.Has(StringValue("foo"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Has(StringValue("bar"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - ASSERT_THAT(map.Has(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); } TEST_F(CustomMapValueTest, Dispatcher_Has_Error) { CustomMapValue map = MakeDispatcher(); Value result; - ASSERT_THAT(map.Has(StringValue("error"), descriptor_pool(), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena(), &result), IsOk()); EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, "custom error"))); - EXPECT_THAT(map.Has(StringValue("error"), descriptor_pool(), + EXPECT_THAT(map.Has(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs( absl::StatusCode::kInvalidArgument, "custom error")))); @@ -695,12 +695,12 @@ TEST_F(CustomMapValueTest, Dispatcher_Has_Error) { TEST_F(CustomMapValueTest, Interface_Has_Error) { CustomMapValue map = MakeInterface(); Value result; - ASSERT_THAT(map.Has(StringValue("error"), descriptor_pool(), + ASSERT_THAT(map.Has(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena(), &result), IsOk()); EXPECT_THAT(result, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument, "custom error"))); - EXPECT_THAT(map.Has(StringValue("error"), descriptor_pool(), + EXPECT_THAT(map.Has(StringValue::WrapUnsafe("error"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs( absl::StatusCode::kInvalidArgument, "custom error")))); diff --git a/common/values/legacy_struct_value_test.cc b/common/values/legacy_struct_value_test.cc index 2e1949c0b..1329232f8 100644 --- a/common/values/legacy_struct_value_test.cc +++ b/common/values/legacy_struct_value_test.cc @@ -354,8 +354,8 @@ TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedJsonList) { auto list_val = val.GetList(); EXPECT_THAT(list_val.IsEmpty(), IsOkAndHolds(false)); EXPECT_THAT(list_val.Size(), IsOkAndHolds(2)); - EXPECT_THAT(list_val.Contains(StringValue("item1"), descriptor_pool(), - message_factory(), arena()), + EXPECT_THAT(list_val.Contains(StringValue::WrapUnsafe("item1"), + descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); Value elem; @@ -403,10 +403,10 @@ TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedMapField) { EXPECT_THAT(map_val.IsEmpty(), IsOkAndHolds(false)); EXPECT_FALSE(map_val.IsZeroValue()); EXPECT_THAT(map_val.Size(), IsOkAndHolds(2)); - EXPECT_THAT(map_val.Has(StringValue("key1"), descriptor_pool(), + EXPECT_THAT(map_val.Has(StringValue::WrapUnsafe("key1"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(map_val.Has(StringValue("missing"), descriptor_pool(), + EXPECT_THAT(map_val.Has(StringValue::WrapUnsafe("missing"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); EXPECT_THAT( @@ -414,13 +414,13 @@ TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedMapField) { IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); Value found_val; - ASSERT_THAT(map_val.Find(StringValue("key1"), descriptor_pool(), + ASSERT_THAT(map_val.Find(StringValue::WrapUnsafe("key1"), descriptor_pool(), message_factory(), arena(), &found_val), IsOkAndHolds(true)); EXPECT_THAT(found_val, StringValueIs("val1")); Value get_val; - ASSERT_THAT(map_val.Get(StringValue("key2"), descriptor_pool(), + ASSERT_THAT(map_val.Get(StringValue::WrapUnsafe("key2"), descriptor_pool(), message_factory(), arena(), &get_val), IsOk()); EXPECT_THAT(get_val, StringValueIs("val2")); @@ -466,10 +466,10 @@ TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedJsonMap) { EXPECT_THAT(map_val.IsEmpty(), IsOkAndHolds(false)); EXPECT_FALSE(map_val.IsZeroValue()); EXPECT_THAT(map_val.Size(), IsOkAndHolds(2)); - EXPECT_THAT(map_val.Has(StringValue("k1"), descriptor_pool(), + EXPECT_THAT(map_val.Has(StringValue::WrapUnsafe("k1"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(map_val.Has(StringValue("missing"), descriptor_pool(), + EXPECT_THAT(map_val.Has(StringValue::WrapUnsafe("missing"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); EXPECT_THAT( @@ -477,13 +477,13 @@ TEST_F(LegacyStructValueTest, WrapLegacyFieldAccessResultParsedJsonMap) { IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); Value found_val; - ASSERT_THAT(map_val.Find(StringValue("k1"), descriptor_pool(), + ASSERT_THAT(map_val.Find(StringValue::WrapUnsafe("k1"), descriptor_pool(), message_factory(), arena(), &found_val), IsOkAndHolds(true)); EXPECT_THAT(found_val, StringValueIs("v1")); Value get_val; - ASSERT_THAT(map_val.Get(StringValue("k2"), descriptor_pool(), + ASSERT_THAT(map_val.Get(StringValue::WrapUnsafe("k2"), descriptor_pool(), message_factory(), arena(), &get_val), IsOk()); EXPECT_THAT(get_val, StringValueIs("v2")); diff --git a/common/values/map_value_test.cc b/common/values/map_value_test.cc index f7d1c5197..a1a82fd03 100644 --- a/common/values/map_value_test.cc +++ b/common/values/map_value_test.cc @@ -270,9 +270,10 @@ TEST_F(MapValueTest, NewIterator) { TEST_F(MapValueTest, ConvertToJson) { ASSERT_OK_AND_ASSIGN( auto value, - NewJsonMapValue(std::pair{StringValue("0"), DoubleValue(3.0)}, - std::pair{StringValue("1"), DoubleValue(4.0)}, - std::pair{StringValue("2"), DoubleValue(5.0)})); + NewJsonMapValue( + std::pair{StringValue::WrapUnsafe("0"), DoubleValue(3.0)}, + std::pair{StringValue::WrapUnsafe("1"), DoubleValue(4.0)}, + std::pair{StringValue::WrapUnsafe("2"), DoubleValue(5.0)})); auto* message = NewArenaValueMessage(); EXPECT_THAT( value.ConvertToJson(descriptor_pool(), message_factory(), message), diff --git a/common/values/mutable_list_value_test.cc b/common/values/mutable_list_value_test.cc index c08d7091c..c6aec0df4 100644 --- a/common/values/mutable_list_value_test.cc +++ b/common/values/mutable_list_value_test.cc @@ -47,7 +47,8 @@ TEST_F(MutableListValueTest, IsEmpty) { auto* mutable_list_value = NewMutableListValue(arena()); mutable_list_value->Reserve(1); EXPECT_TRUE(CustomListValue(mutable_list_value, arena()).IsEmpty()); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); EXPECT_FALSE(CustomListValue(mutable_list_value, arena()).IsEmpty()); } @@ -55,7 +56,8 @@ TEST_F(MutableListValueTest, Size) { auto* mutable_list_value = NewMutableListValue(arena()); mutable_list_value->Reserve(1); EXPECT_THAT(CustomListValue(mutable_list_value, arena()).Size(), 0); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); EXPECT_THAT(CustomListValue(mutable_list_value, arena()).Size(), 1); } @@ -73,7 +75,8 @@ TEST_F(MutableListValueTest, ForEach) { message_factory(), arena()), IsOk()); EXPECT_THAT(elements, IsEmpty()); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); EXPECT_THAT(CustomListValue(mutable_list_value, arena()) .ForEach(for_each_callback, descriptor_pool(), message_factory(), arena()), @@ -89,7 +92,8 @@ TEST_F(MutableListValueTest, NewIterator) { CustomListValue(mutable_list_value, arena()).NewIterator()); EXPECT_THAT(iterator->Next(descriptor_pool(), message_factory(), arena()), StatusIs(absl::StatusCode::kFailedPrecondition)); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); ASSERT_OK_AND_ASSIGN( iterator, CustomListValue(mutable_list_value, arena()).NewIterator()); EXPECT_TRUE(iterator->HasNext()); @@ -110,7 +114,8 @@ TEST_F(MutableListValueTest, Get) { IsOk()); EXPECT_THAT(value, ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))); - EXPECT_THAT(mutable_list_value->Append(StringValue("foo")), IsOk()); + EXPECT_THAT(mutable_list_value->Append(StringValue::WrapUnsafe("foo")), + IsOk()); EXPECT_THAT( CustomListValue(mutable_list_value, arena()) .Get(0, descriptor_pool(), message_factory(), arena(), &value), diff --git a/common/values/mutable_map_value_test.cc b/common/values/mutable_map_value_test.cc index 2f08abe3f..c03e9cbd5 100644 --- a/common/values/mutable_map_value_test.cc +++ b/common/values/mutable_map_value_test.cc @@ -52,7 +52,9 @@ TEST_F(MutableMapValueTest, IsEmpty) { auto mutable_map_value = NewMutableMapValue(arena()); mutable_map_value->Reserve(1); EXPECT_TRUE(CustomMapValue(mutable_map_value, arena()).IsEmpty()); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_FALSE(CustomMapValue(mutable_map_value, arena()).IsEmpty()); } @@ -60,7 +62,9 @@ TEST_F(MutableMapValueTest, Size) { auto mutable_map_value = NewMutableMapValue(arena()); mutable_map_value->Reserve(1); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()).Size(), 0); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()).Size(), 1); } @@ -68,7 +72,9 @@ TEST_F(MutableMapValueTest, ListKeys) { auto mutable_map_value = NewMutableMapValue(arena()); mutable_map_value->Reserve(1); ListValue keys; - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_THAT( CustomMapValue(mutable_map_value, arena()) .ListKeys(descriptor_pool(), message_factory(), arena(), &keys), @@ -92,7 +98,9 @@ TEST_F(MutableMapValueTest, ForEach) { message_factory(), arena()), IsOk()); EXPECT_THAT(entries, IsEmpty()); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) .ForEach(for_each_callback, descriptor_pool(), message_factory(), arena()), @@ -109,7 +117,9 @@ TEST_F(MutableMapValueTest, NewIterator) { EXPECT_FALSE(iterator->HasNext()); EXPECT_THAT(iterator->Next(descriptor_pool(), message_factory(), arena()), StatusIs(absl::StatusCode::kFailedPrecondition)); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); ASSERT_OK_AND_ASSIGN( iterator, CustomMapValue(mutable_map_value, arena()).NewIterator()); EXPECT_TRUE(iterator->HasNext()); @@ -125,24 +135,26 @@ TEST_F(MutableMapValueTest, FindHas) { mutable_map_value->Reserve(1); Value value; EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) - .Find(StringValue("foo"), descriptor_pool(), + .Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena(), &value), IsOkAndHolds(IsFalse())); EXPECT_THAT(value, IsNullValue()); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) - .Has(StringValue("foo"), descriptor_pool(), message_factory(), - arena(), &value), + .Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena(), &value), IsOk()); EXPECT_THAT(value, BoolValueIs(false)); - EXPECT_THAT(mutable_map_value->Put(StringValue("foo"), IntValue(1)), IsOk()); + EXPECT_THAT( + mutable_map_value->Put(StringValue::WrapUnsafe("foo"), IntValue(1)), + IsOk()); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) - .Find(StringValue("foo"), descriptor_pool(), + .Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena(), &value), IsOkAndHolds(IsTrue())); EXPECT_THAT(value, IntValueIs(1)); EXPECT_THAT(CustomMapValue(mutable_map_value, arena()) - .Has(StringValue("foo"), descriptor_pool(), message_factory(), - arena(), &value), + .Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena(), &value), IsOk()); EXPECT_THAT(value, BoolValueIs(true)); } diff --git a/common/values/parsed_json_list_value_test.cc b/common/values/parsed_json_list_value_test.cc index 5fcff20bf..7551463f4 100644 --- a/common/values/parsed_json_list_value_test.cc +++ b/common/values/parsed_json_list_value_test.cc @@ -251,12 +251,14 @@ TEST_F(ParsedJsonListValueTest, Contains_Dynamic) { EXPECT_THAT(valid_value.Contains(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(valid_value.Contains(StringValue("bar"), descriptor_pool(), - message_factory(), arena()), - IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(valid_value.Contains(StringValue("foo"), descriptor_pool(), - message_factory(), arena()), - IsOkAndHolds(BoolValueIs(true))); + EXPECT_THAT( + valid_value.Contains(StringValue::WrapUnsafe("bar"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(false))); + EXPECT_THAT( + valid_value.Contains(StringValue::WrapUnsafe("foo"), descriptor_pool(), + message_factory(), arena()), + IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT(valid_value.Contains( ParsedJsonListValue( DynamicParseTextProto( diff --git a/common/values/parsed_json_map_value.cc b/common/values/parsed_json_map_value.cc index ec8c91a4f..ad0ad558c 100644 --- a/common/values/parsed_json_map_value.cc +++ b/common/values/parsed_json_map_value.cc @@ -325,7 +325,7 @@ absl::Status ParsedJsonMapValue::ForEach( const auto map_end = reflection.EndFields(*value_); for (; map_begin != map_end; ++map_begin) { // We have to copy until `google::protobuf::MapKey` is just a view. - key_scratch = StringValue(arena, map_begin.GetKey().GetStringValue()); + key_scratch = StringValue::From(map_begin.GetKey().GetStringValue(), arena); value_scratch = common_internal::ParsedJsonValue( &map_begin.GetValueRef().GetMessageValue(), arena); CEL_ASSIGN_OR_RETURN(auto ok, callback(key_scratch, value_scratch)); diff --git a/common/values/parsed_json_map_value_test.cc b/common/values/parsed_json_map_value_test.cc index 67bddcf8e..8dfca0c70 100644 --- a/common/values/parsed_json_map_value_test.cc +++ b/common/values/parsed_json_map_value_test.cc @@ -144,15 +144,15 @@ TEST_F(ParsedJsonMapValueTest, Get_Dynamic) { valid_value.Get(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); - EXPECT_THAT(valid_value.Get(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(valid_value.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(IsNullValue())); - EXPECT_THAT(valid_value.Get(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(valid_value.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT( - valid_value.Get(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + valid_value.Get(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); } @@ -171,14 +171,14 @@ TEST_F(ParsedJsonMapValueTest, Find_Dynamic) { EXPECT_THAT(valid_value.Find(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Eq(std::nullopt))); - EXPECT_THAT(valid_value.Find(StringValue("foo"), descriptor_pool(), - message_factory(), arena()), + EXPECT_THAT(valid_value.Find(StringValue::WrapUnsafe("foo"), + descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(IsNullValue()))); - EXPECT_THAT(valid_value.Find(StringValue("bar"), descriptor_pool(), - message_factory(), arena()), + EXPECT_THAT(valid_value.Find(StringValue::WrapUnsafe("bar"), + descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); - EXPECT_THAT(valid_value.Find(StringValue("baz"), descriptor_pool(), - message_factory(), arena()), + EXPECT_THAT(valid_value.Find(StringValue::WrapUnsafe("baz"), + descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Eq(std::nullopt))); } @@ -197,13 +197,13 @@ TEST_F(ParsedJsonMapValueTest, Has_Dynamic) { EXPECT_THAT(valid_value.Has(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(valid_value.Has(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(valid_value.Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(valid_value.Has(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(valid_value.Has(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(valid_value.Has(StringValue("baz"), descriptor_pool(), + EXPECT_THAT(valid_value.Has(StringValue::WrapUnsafe("baz"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); } @@ -229,7 +229,7 @@ TEST_F(ParsedJsonMapValueTest, ListKeys_Dynamic) { EXPECT_THAT( keys.Contains(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(keys.Contains(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(keys.Contains(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT(keys.Get(0, descriptor_pool(), message_factory(), arena()), @@ -379,10 +379,10 @@ TEST_F(ParsedJsonMapValueTest, CloneDifferentArena) { cloned.Equal(value, descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_EQ(cloned.Size(), 2); - EXPECT_THAT(cloned.Get(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(cloned.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(IsNullValue())); - EXPECT_THAT(cloned.Get(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(cloned.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); } diff --git a/common/values/parsed_json_value.cc b/common/values/parsed_json_value.cc index 24ba36645..fdcd5a1d0 100644 --- a/common/values/parsed_json_value.cc +++ b/common/values/parsed_json_value.cc @@ -25,8 +25,6 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/variant.h" -#include "common/allocator.h" -#include "common/memory.h" #include "common/value.h" #include "internal/well_known_types.h" #include "google/protobuf/arena.h" @@ -74,17 +72,20 @@ Value ParsedJsonValue(const google::protobuf::Message* absl_nonnull message, } if (string.data() == scratch.data() && string.size() == scratch.size()) { - return StringValue(arena, std::move(scratch)); + return StringValue::From(std::move(scratch), arena); } else { - return StringValue( - Borrower::Arena(MessageArenaOr(message, arena)), string); + if (google::protobuf::Arena* message_arena = message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); } }, [&](absl::Cord&& cord) -> StringValue { if (cord.empty()) { return StringValue(); } - return StringValue(std::move(cord)); + return StringValue::From(std::move(cord), arena); }), AsVariant(reflection.GetStringValue(*message, scratch))); } diff --git a/common/values/parsed_map_field_value_test.cc b/common/values/parsed_map_field_value_test.cc index 0fdea333b..f605b5c98 100644 --- a/common/values/parsed_map_field_value_test.cc +++ b/common/values/parsed_map_field_value_test.cc @@ -210,15 +210,15 @@ TEST_F(ParsedMapFieldValueTest, Get) { EXPECT_THAT( value.Get(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); - EXPECT_THAT(value.Get(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(value.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(value.Get(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(value.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT( - value.Get(StringValue("baz"), descriptor_pool(), message_factory(), - arena()), + value.Get(StringValue::WrapUnsafe("baz"), descriptor_pool(), + message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); } @@ -232,13 +232,13 @@ TEST_F(ParsedMapFieldValueTest, Find) { EXPECT_THAT( value.Find(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Eq(std::nullopt))); - EXPECT_THAT(value.Find(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(value.Find(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(false)))); - EXPECT_THAT(value.Find(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(value.Find(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Optional(BoolValueIs(true)))); - EXPECT_THAT(value.Find(StringValue("baz"), descriptor_pool(), + EXPECT_THAT(value.Find(StringValue::WrapUnsafe("baz"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(Eq(std::nullopt))); } @@ -325,13 +325,13 @@ TEST_F(ParsedMapFieldValueTest, Has) { EXPECT_THAT( value.Has(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(value.Has(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(value.Has(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(value.Has(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(value.Has(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); - EXPECT_THAT(value.Has(StringValue("baz"), descriptor_pool(), + EXPECT_THAT(value.Has(StringValue::WrapUnsafe("baz"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); } @@ -351,7 +351,7 @@ TEST_F(ParsedMapFieldValueTest, ListKeys) { EXPECT_THAT( keys.Contains(BoolValue(), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(keys.Contains(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(keys.Contains(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_THAT(keys.Get(0, descriptor_pool(), message_factory(), arena()), @@ -691,10 +691,10 @@ TEST_F(ParsedMapFieldValueTest, CloneDifferentArena) { cloned.Equal(value, descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(true))); EXPECT_EQ(cloned.Size(), 2); - EXPECT_THAT(cloned.Get(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(cloned.Get(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(StringValueIs("bar"))); - EXPECT_THAT(cloned.Get(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(cloned.Get(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(StringValueIs("foo"))); } diff --git a/common/values/parsed_repeated_field_value_test.cc b/common/values/parsed_repeated_field_value_test.cc index a3a80456a..bb50bee55 100644 --- a/common/values/parsed_repeated_field_value_test.cc +++ b/common/values/parsed_repeated_field_value_test.cc @@ -438,10 +438,10 @@ TEST_F(ParsedRepeatedFieldValueTest, Contains) { EXPECT_THAT(value.Contains(DoubleValue(1.0), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(value.Contains(StringValue("bar"), descriptor_pool(), + EXPECT_THAT(value.Contains(StringValue::WrapUnsafe("bar"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); - EXPECT_THAT(value.Contains(StringValue("foo"), descriptor_pool(), + EXPECT_THAT(value.Contains(StringValue::WrapUnsafe("foo"), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(BoolValueIs(false))); EXPECT_THAT( diff --git a/common/values/string_value.cc b/common/values/string_value.cc index 9ebde8795..7659e9a71 100644 --- a/common/values/string_value.cc +++ b/common/values/string_value.cc @@ -35,7 +35,6 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "common/internal/byte_string.h" -#include "common/internal/reference_count.h" #include "common/value.h" #include "internal/status_macros.h" #include "internal/strings.h" @@ -618,8 +617,7 @@ absl::StatusOr SubstringImpl(absl::string_view string, uint64_t start) { ".substring(): is greater than .size()"); } -absl::StatusOr SubstringImpl(const absl::Cord& cord, - uint64_t start) { +absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start) { absl::Cord::CharIterator char_begin = cord.char_begin(); absl::Cord::CharIterator char_end = cord.char_end(); size_t size_code_points = 0; @@ -629,14 +627,14 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, size_t code_units; std::tie(code_point, code_units) = cel::internal::Utf8Decode(char_begin); if (size_code_points == start) { - return cord.Subcord(size_code_units, std::numeric_limits::max()); + return size_code_units; } absl::Cord::Advance(&char_begin, code_units); ++size_code_points; size_code_units += code_units; } if (size_code_points == start) { - return cord; + return size_code_units; } return absl::InvalidArgumentError( ".substring(): is greater than .size()"); @@ -685,17 +683,18 @@ Value StringValue::Substring(int64_t start) const { value_.rep_.medium.size - *status_or_index; result.value_.rep_.medium.data = value_.rep_.medium.data + *status_or_index; - result.value_.rep_.medium.owner = value_.rep_.medium.owner; - common_internal::StrongRef(result.value_.GetMediumReferenceCount()); + result.value_.rep_.medium.arena = value_.rep_.medium.arena; return result; } case common_internal::ByteStringKind::kLarge: { - absl::StatusOr status_or_cord = + absl::StatusOr status_or_index = (SubstringImpl)(value_.GetLarge(), start); - if (!status_or_cord.ok()) { - return ErrorValue(std::move(status_or_cord).status()); + if (!status_or_index.ok()) { + return ErrorValue(std::move(status_or_index).status()); } - return StringValue::Wrap(*std::move(status_or_cord)); + return StringValue(common_internal::ByteString::Wrap( + value_.rep_.large.data, value_.rep_.large.offset + *status_or_index, + value_.rep_.large.size - *status_or_index, value_.rep_.large.arena)); } } } @@ -729,8 +728,9 @@ absl::StatusOr> SubstringImpl( ".size()"); } -absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start, - uint64_t end) { +absl::StatusOr> SubstringImpl(const absl::Cord& cord, + uint64_t start, + uint64_t end) { absl::Cord::CharIterator char_begin = cord.char_begin(); absl::Cord::CharIterator char_end = cord.char_end(); size_t size_code_points = 0; @@ -741,7 +741,7 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start, start_code_units = size_code_units; } if (size_code_points == end) { - return cord.Subcord(start_code_units, size_code_units - start_code_units); + return std::pair{start_code_units, size_code_units}; } char32_t code_point; size_t code_units; @@ -751,7 +751,7 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start, size_code_units += code_units; } if (size_code_points == start && start == end) { - return absl::Cord(); + return std::pair{size_code_units, size_code_units}; } return absl::InvalidArgumentError( ".substring(, ): or is greater than " @@ -804,17 +804,20 @@ Value StringValue::Substring(int64_t start, int64_t end) const { (status_or_indices->second - status_or_indices->first); result.value_.rep_.medium.data = value_.rep_.medium.data + status_or_indices->first; - result.value_.rep_.medium.owner = value_.rep_.medium.owner; - common_internal::StrongRef(result.value_.GetMediumReferenceCount()); + result.value_.rep_.medium.arena = value_.rep_.medium.arena; return result; } case common_internal::ByteStringKind::kLarge: { - absl::StatusOr status_or_cord = + absl::StatusOr> status_or_indices = (SubstringImpl)(value_.GetLarge(), start, end); - if (!status_or_cord.ok()) { - return ErrorValue(std::move(status_or_cord).status()); + if (!status_or_indices.ok()) { + return ErrorValue(std::move(status_or_indices).status()); } - return StringValue::Wrap(*std::move(status_or_cord)); + return StringValue(common_internal::ByteString::Wrap( + value_.rep_.large.data, + value_.rep_.large.offset + status_or_indices->first, + status_or_indices->second - status_or_indices->first, + value_.rep_.large.arena)); } } } @@ -841,9 +844,9 @@ bool LowerAsciiImpl(absl::string_view in, std::string* absl_nonnull out) { return true; } -absl::Cord LowerAsciiImpl(const absl::Cord& in) { +bool LowerAsciiImpl(const absl::Cord& in, absl::Cord* absl_nonnull out) { if (in.empty()) { - return in; + return false; } size_t pos = 0; bool needs_conversion = false; @@ -855,9 +858,9 @@ absl::Cord LowerAsciiImpl(const absl::Cord& in) { pos++; } if (!needs_conversion) { - return in; + return false; } - absl::Cord out = in.Subcord(0, pos); + absl::Cord prefix = in.Subcord(0, pos); absl::Cord rest = in.Subcord(pos, in.size() - pos); std::string suffix; suffix.resize(rest.size()); @@ -865,8 +868,9 @@ absl::Cord LowerAsciiImpl(const absl::Cord& in) { for (char c : rest.Chars()) { suffix[current++] = absl::ascii_tolower(c); } - out.Append(std::move(suffix)); - return out; + prefix.Append(std::move(suffix)); + *out = std::move(prefix); + return true; } } // namespace @@ -889,8 +893,13 @@ StringValue StringValue::LowerAscii(google::protobuf::Arena* absl_nonnull arena) } return StringValue::From(std::move(out), arena); } - case common_internal::ByteStringKind::kLarge: - return StringValue::Wrap((LowerAsciiImpl)(value_.GetLarge())); + case common_internal::ByteStringKind::kLarge: { + absl::Cord out; + if (!(LowerAsciiImpl)(value_.GetLarge(), &out)) { + return *this; + } + return StringValue::From(std::move(out), arena); + } } } @@ -916,9 +925,9 @@ bool UpperAsciiImpl(absl::string_view in, std::string* absl_nonnull out) { return true; } -absl::Cord UpperAsciiImpl(const absl::Cord& in) { +bool UpperAsciiImpl(const absl::Cord& in, absl::Cord* absl_nonnull out) { if (in.empty()) { - return in; + return false; } size_t pos = 0; bool needs_conversion = false; @@ -930,9 +939,9 @@ absl::Cord UpperAsciiImpl(const absl::Cord& in) { pos++; } if (!needs_conversion) { - return in; + return false; } - absl::Cord out = in.Subcord(0, pos); + absl::Cord prefix = in.Subcord(0, pos); absl::Cord rest = in.Subcord(pos, in.size() - pos); std::string suffix; suffix.resize(rest.size()); @@ -940,15 +949,15 @@ absl::Cord UpperAsciiImpl(const absl::Cord& in) { for (char c : rest.Chars()) { suffix[current++] = absl::ascii_toupper(c); } - out.Append(std::move(suffix)); - return out; + prefix.Append(std::move(suffix)); + *out = std::move(prefix); + return true; } } // namespace StringValue StringValue::UpperAscii(google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); - switch (value_.GetKind()) { case common_internal::ByteStringKind::kSmall: { std::string out; @@ -964,8 +973,13 @@ StringValue StringValue::UpperAscii(google::protobuf::Arena* absl_nonnull arena) } return StringValue::From(std::move(out), arena); } - case common_internal::ByteStringKind::kLarge: - return StringValue::Wrap((UpperAsciiImpl)(value_.GetLarge())); + case common_internal::ByteStringKind::kLarge: { + absl::Cord out; + if (!(UpperAsciiImpl)(value_.GetLarge(), &out)) { + return *this; + } + return StringValue::From(std::move(out), arena); + } } } @@ -1016,7 +1030,7 @@ std::pair TrimImpl(absl::string_view string) { return {left_trim_bytes, string.size() - last_non_ws_end_bytes}; } -absl::Cord TrimImpl(const absl::Cord& cord) { +std::pair TrimImpl(const absl::Cord& cord) { size_t left_trim_bytes = 0; { absl::Cord::CharIterator begin = cord.char_begin(); @@ -1034,7 +1048,7 @@ absl::Cord TrimImpl(const absl::Cord& cord) { } if (left_trim_bytes == cord.size()) { - return absl::Cord(); + return {left_trim_bytes, 0}; } absl::Cord ltrimmed = @@ -1056,40 +1070,29 @@ absl::Cord TrimImpl(const absl::Cord& cord) { current_pos_bytes += char_len; } } - return ltrimmed.Subcord(0, last_non_ws_end_bytes); + return {left_trim_bytes, ltrimmed.size() - last_non_ws_end_bytes}; } } // namespace StringValue StringValue::Trim() const { + std::pair trims; + size_t size; switch (value_.GetKind()) { - case common_internal::ByteStringKind::kSmall: { - std::pair trims = (TrimImpl)(value_.GetSmall()); - StringValue result; - result.value_.rep_.header.kind = common_internal::ByteStringKind::kSmall; - result.value_.rep_.small.size = - value_.rep_.small.size - trims.first - trims.second; - std::memcpy(result.value_.rep_.small.data, - value_.rep_.small.data + trims.first, - result.value_.rep_.small.size); - result.value_.rep_.small.arena = value_.GetSmallArena(); - return result; - } - case common_internal::ByteStringKind::kMedium: { - std::pair trims = (TrimImpl)(value_.GetMedium()); - StringValue result; - result.value_.rep_.header.kind = common_internal::ByteStringKind::kMedium; - result.value_.rep_.medium.size = - value_.rep_.medium.size - trims.first - trims.second; - result.value_.rep_.medium.data = value_.rep_.medium.data + trims.first; - result.value_.rep_.medium.owner = value_.rep_.medium.owner; - common_internal::StrongRef(result.value_.GetMediumReferenceCount()); - return result; - } - case common_internal::ByteStringKind::kLarge: { - return StringValue::Wrap((TrimImpl)(value_.GetLarge())); - } + case common_internal::ByteStringKind::kSmall: + trims = (TrimImpl)(value_.GetSmall()); + size = value_.GetSmall().size(); + break; + case common_internal::ByteStringKind::kMedium: + trims = (TrimImpl)(value_.GetMedium()); + size = value_.GetMedium().size(); + break; + case common_internal::ByteStringKind::kLarge: + trims = (TrimImpl)(value_.GetLarge()); + size = value_.rep_.large.size; + break; } + return StringValue(value_.Substring(trims.first, size - trims.second)); } namespace { diff --git a/common/values/string_value.h b/common/values/string_value.h index 26cdec6c1..aa478ce84 100644 --- a/common/values/string_value.h +++ b/common/values/string_value.h @@ -28,16 +28,12 @@ #include "absl/base/attributes.h" #include "absl/base/macros.h" #include "absl/base/nullability.h" -#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" -#include "common/arena.h" #include "common/internal/byte_string.h" -#include "common/memory.h" #include "common/type.h" #include "common/value_kind.h" #include "common/values/values.h" @@ -82,28 +78,48 @@ class StringValue final : private common_internal::ValueMixin { static StringValue From(const char* absl_nullable value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::From(value, arena)); + } static StringValue From(absl::string_view value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static StringValue From(const absl::Cord& value); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::From(value, arena)); + } + static StringValue From(const absl::Cord& value, + google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::From(value, arena)); + } static StringValue From(std::string&& value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue( + common_internal::ByteString::From(std::move(value), arena)); + } static StringValue Wrap(absl::string_view value, google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); - static StringValue Wrap(absl::string_view value) = delete; - static StringValue Wrap(const absl::Cord& value); - static StringValue Wrap(std::string&& value) = delete; - static StringValue Wrap(std::string&& value, - google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) = delete; + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::Wrap(value, arena)); + } + static StringValue Wrap( + const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return StringValue(common_internal::ByteString::Wrap(value, arena)); + } + static StringValue Wrap(std::nullptr_t, google::protobuf::Arena*) = delete; + static StringValue Wrap(std::string&& value, google::protobuf::Arena*) = delete; // Returns a StringValue that aliases the provided string. Caller must ensure // the provided string outlives the use of the returned StringValue. - static StringValue WrapUnsafe(absl::string_view value); + static StringValue WrapUnsafe(absl::string_view value) { + return StringValue(common_internal::ByteString::WrapUnsafe(value)); + } + static StringValue WrapUnsafe(const absl::Cord* absl_nonnull value) { + return StringValue(common_internal::ByteString::WrapUnsafe(value)); + } + static StringValue WrapUnsafe(std::nullptr_t) = delete; // Returns a StringValue that aliases the provided string, which must be a // compile time constant. This is enforced during compilation when using @@ -126,48 +142,14 @@ class StringValue final : private common_internal::ValueMixin { google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND); - ABSL_DEPRECATED("Use From") - explicit StringValue(const char* absl_nullable value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit StringValue(absl::string_view value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit StringValue(const absl::Cord& value) : value_(value) {} - - ABSL_DEPRECATED("Use From") - explicit StringValue(std::string&& value) : value_(std::move(value)) {} - - ABSL_DEPRECATED("Use From") - StringValue(Allocator<> allocator, const char* absl_nullable value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - StringValue(Allocator<> allocator, absl::string_view value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - StringValue(Allocator<> allocator, const absl::Cord& value) - : value_(allocator, value) {} - - ABSL_DEPRECATED("Use From") - StringValue(Allocator<> allocator, std::string&& value) - : value_(allocator, std::move(value)) {} - - ABSL_DEPRECATED("Use Wrap") - StringValue(Borrower borrower, absl::string_view value) - : value_(borrower, value) {} - - ABSL_DEPRECATED("Use Wrap") - StringValue(Borrower borrower, const absl::Cord& value) - : value_(borrower, value) {} - StringValue() = default; StringValue(const StringValue&) = default; StringValue(StringValue&&) = default; StringValue& operator=(const StringValue&) = default; StringValue& operator=(StringValue&&) = default; + explicit StringValue(const BytesValue& other); + constexpr ValueKind kind() const { return kKind; } absl::string_view GetTypeName() const { return StringType::kName; } @@ -222,9 +204,9 @@ class StringValue final : private common_internal::ValueMixin { return value_.Visit(std::forward(visitor)); } - void swap(StringValue& other) noexcept { + friend void swap(StringValue& lhs, StringValue& rhs) noexcept { using std::swap; - swap(value_, other.value_); + swap(lhs.value_, rhs.value_); } size_t Size() const; @@ -390,10 +372,10 @@ class StringValue final : private common_internal::ValueMixin { } private: + friend class BytesValue; friend class common_internal::ValueMixin; friend absl::string_view common_internal::LegacyStringValue( const StringValue& value, bool stable, google::protobuf::Arena* absl_nonnull arena); - friend struct ArenaTraits; explicit StringValue(common_internal::ByteString value) noexcept : value_(std::move(value)) {} @@ -401,8 +383,6 @@ class StringValue final : private common_internal::ValueMixin { common_internal::ByteString value_; }; -inline void swap(StringValue& lhs, StringValue& rhs) noexcept { lhs.swap(rhs); } - inline bool operator==(const StringValue& lhs, absl::string_view rhs) { return lhs.Equals(rhs); } @@ -459,48 +439,6 @@ inline std::ostream& operator<<(std::ostream& out, const StringValue& value) { return out << value.DebugString(); } -inline StringValue StringValue::From(const char* absl_nullable value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return From(absl::NullSafeStringView(value), arena); -} - -inline StringValue StringValue::From(absl::string_view value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return StringValue(arena, value); -} - -inline StringValue StringValue::From(const absl::Cord& value) { - return StringValue(value); -} - -inline StringValue StringValue::From(std::string&& value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return StringValue(arena, std::move(value)); -} - -inline StringValue StringValue::Wrap(absl::string_view value, - google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) { - ABSL_DCHECK(arena != nullptr); - - return StringValue(Borrower::Arena(arena), value); -} - -inline StringValue StringValue::WrapUnsafe(absl::string_view value) { - return StringValue(common_internal::ByteString::FromExternal(value)); -} - -inline StringValue StringValue::Wrap(const absl::Cord& value) { - return StringValue(value); -} - namespace common_internal { inline absl::string_view LegacyStringValue(const StringValue& value, @@ -511,15 +449,6 @@ inline absl::string_view LegacyStringValue(const StringValue& value, } // namespace common_internal -template <> -struct ArenaTraits { - using constructible = std::true_type; - - static bool trivially_destructible(const StringValue& value) { - return ArenaTraits<>::trivially_destructible(value.value_); - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_STRING_VALUE_H_ diff --git a/common/values/string_value_test.cc b/common/values/string_value_test.cc index 95d72fc65..be21e4d1e 100644 --- a/common/values/string_value_test.cc +++ b/common/values/string_value_test.cc @@ -40,185 +40,212 @@ using ::testing::Optional; using StringValueTest = common_internal::ValueTest<>; TEST_F(StringValueTest, Kind) { - EXPECT_EQ(StringValue("foo").kind(), StringValue::kKind); - EXPECT_EQ(Value(StringValue(absl::Cord("foo"))).kind(), StringValue::kKind); + EXPECT_EQ(StringValue::WrapUnsafe("foo").kind(), StringValue::kKind); + EXPECT_EQ(Value(StringValue::From(absl::Cord("foo"), arena())).kind(), + StringValue::kKind); } TEST_F(StringValueTest, DebugString) { { std::ostringstream out; - out << StringValue("foo"); + out << StringValue::WrapUnsafe("foo"); EXPECT_EQ(out.str(), "\"foo\""); } { std::ostringstream out; - out << StringValue(absl::MakeFragmentedCord({"f", "o", "o"})); + out << StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), + arena()); EXPECT_EQ(out.str(), "\"foo\""); } { std::ostringstream out; - out << Value(StringValue(absl::Cord("foo"))); + out << Value(StringValue::From(absl::Cord("foo"), arena())); EXPECT_EQ(out.str(), "\"foo\""); } } TEST_F(StringValueTest, ConvertToJson) { auto* message = NewArenaValueMessage(); - EXPECT_THAT(StringValue("foo").ConvertToJson(descriptor_pool(), - message_factory(), message), + EXPECT_THAT(StringValue::WrapUnsafe("foo").ConvertToJson( + descriptor_pool(), message_factory(), message), IsOk()); EXPECT_THAT(*message, EqualsValueTextProto(R"pb(string_value: "foo")pb")); } TEST_F(StringValueTest, NativeValue) { std::string scratch; - EXPECT_EQ(StringValue("foo").NativeString(), "foo"); - EXPECT_EQ(StringValue("foo").NativeString(scratch), "foo"); - EXPECT_EQ(StringValue("foo").NativeCord(), "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").NativeString(), "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").NativeString(scratch), "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").NativeCord(), "foo"); } TEST_F(StringValueTest, TryFlat) { - EXPECT_THAT(StringValue("foo").TryFlat(), Optional(Eq("foo"))); + EXPECT_THAT(StringValue::WrapUnsafe("foo").TryFlat(), Optional(Eq("foo"))); EXPECT_THAT( - StringValue(absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"})) + StringValue::From( + absl::MakeFragmentedCord({"Hello, World!", "World, Hello!"}), arena()) .TryFlat(), Eq(std::nullopt)); } TEST_F(StringValueTest, ToString) { - EXPECT_EQ(StringValue("foo").ToString(), "foo"); - EXPECT_EQ(StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).ToString(), - "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").ToString(), "foo"); + EXPECT_EQ( + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .ToString(), + "foo"); } TEST_F(StringValueTest, CopyToString) { std::string out; - StringValue("foo").CopyToString(&out); + StringValue::WrapUnsafe("foo").CopyToString(&out); EXPECT_EQ(out, "foo"); - StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).CopyToString(&out); + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .CopyToString(&out); EXPECT_EQ(out, "foo"); } TEST_F(StringValueTest, AppendToString) { std::string out; - StringValue("foo").AppendToString(&out); + StringValue::WrapUnsafe("foo").AppendToString(&out); EXPECT_EQ(out, "foo"); - StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).AppendToString(&out); + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .AppendToString(&out); EXPECT_EQ(out, "foofoo"); } TEST_F(StringValueTest, ToCord) { - EXPECT_EQ(StringValue("foo").ToCord(), "foo"); - EXPECT_EQ(StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).ToCord(), - "foo"); + EXPECT_EQ(StringValue::WrapUnsafe("foo").ToCord(), "foo"); + EXPECT_EQ( + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .ToCord(), + "foo"); } TEST_F(StringValueTest, CopyToCord) { absl::Cord out; - StringValue("foo").CopyToCord(&out); + StringValue::WrapUnsafe("foo").CopyToCord(&out); EXPECT_EQ(out, "foo"); - StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).CopyToCord(&out); + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .CopyToCord(&out); EXPECT_EQ(out, "foo"); } TEST_F(StringValueTest, AppendToCord) { absl::Cord out; - StringValue("foo").AppendToCord(&out); + StringValue::WrapUnsafe("foo").AppendToCord(&out); EXPECT_EQ(out, "foo"); - StringValue(absl::MakeFragmentedCord({"f", "o", "o"})).AppendToCord(&out); + StringValue::From(absl::MakeFragmentedCord({"f", "o", "o"}), arena()) + .AppendToCord(&out); EXPECT_EQ(out, "foofoo"); } TEST_F(StringValueTest, NativeTypeId) { - EXPECT_EQ(NativeTypeId::Of(StringValue("foo")), - NativeTypeId::For()); - EXPECT_EQ(NativeTypeId::Of(Value(StringValue(absl::Cord("foo")))), + EXPECT_EQ(NativeTypeId::Of(StringValue::WrapUnsafe("foo")), NativeTypeId::For()); + EXPECT_EQ( + NativeTypeId::Of(Value(StringValue::From(absl::Cord("foo"), arena()))), + NativeTypeId::For()); } TEST_F(StringValueTest, HashValue) { - EXPECT_EQ(absl::HashOf(StringValue("foo")), + EXPECT_EQ(absl::HashOf(StringValue::WrapUnsafe("foo")), absl::HashOf(absl::string_view("foo"))); - EXPECT_EQ(absl::HashOf(StringValue(absl::string_view("foo"))), + EXPECT_EQ(absl::HashOf(StringValue::WrapUnsafe(absl::string_view("foo"))), absl::HashOf(absl::string_view("foo"))); - EXPECT_EQ(absl::HashOf(StringValue(absl::Cord("foo"))), + EXPECT_EQ(absl::HashOf(StringValue::From(absl::Cord("foo"), arena())), absl::HashOf(absl::string_view("foo"))); } TEST_F(StringValueTest, Equality) { - EXPECT_NE(StringValue("foo"), "bar"); - EXPECT_NE("bar", StringValue("foo")); - EXPECT_NE(StringValue("foo"), StringValue("bar")); - EXPECT_NE(StringValue("foo"), absl::Cord("bar")); - EXPECT_NE(absl::Cord("bar"), StringValue("foo")); + EXPECT_NE(StringValue::WrapUnsafe("foo"), "bar"); + EXPECT_NE("bar", StringValue::WrapUnsafe("foo")); + EXPECT_NE(StringValue::WrapUnsafe("foo"), StringValue::WrapUnsafe("bar")); + EXPECT_NE(StringValue::WrapUnsafe("foo"), absl::Cord("bar")); + EXPECT_NE(absl::Cord("bar"), StringValue::WrapUnsafe("foo")); } TEST_F(StringValueTest, LessThan) { - EXPECT_LT(StringValue("bar"), "foo"); - EXPECT_LT("bar", StringValue("foo")); - EXPECT_LT(StringValue("bar"), StringValue("foo")); - EXPECT_LT(StringValue("bar"), absl::Cord("foo")); - EXPECT_LT(absl::Cord("bar"), StringValue("foo")); + EXPECT_LT(StringValue::WrapUnsafe("bar"), "foo"); + EXPECT_LT("bar", StringValue::WrapUnsafe("foo")); + EXPECT_LT(StringValue::WrapUnsafe("bar"), StringValue::WrapUnsafe("foo")); + EXPECT_LT(StringValue::WrapUnsafe("bar"), absl::Cord("foo")); + EXPECT_LT(absl::Cord("bar"), StringValue::WrapUnsafe("foo")); } TEST_F(StringValueTest, StartsWith) { EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .StartsWith(StringValue("This string is large enough"))); - EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .StartsWith(StringValue(absl::Cord("This string is large enough")))); + StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .StartsWith(StringValue::WrapUnsafe("This string is large enough"))); + EXPECT_TRUE(StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .StartsWith(StringValue::From( + absl::Cord("This string is large enough"), arena()))); EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .StartsWith(StringValue("This string is large enough"))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .StartsWith(StringValue::WrapUnsafe("This string is large enough"))); EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .StartsWith(StringValue(absl::Cord("This string is large enough")))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .StartsWith(StringValue::From( + absl::Cord("This string is large enough"), arena()))); } TEST_F(StringValueTest, EndsWith) { EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .EndsWith(StringValue("to not be stored inline!"))); + StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .EndsWith(StringValue::WrapUnsafe("to not be stored inline!"))); + EXPECT_TRUE(StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .EndsWith(StringValue::From( + absl::Cord("to not be stored inline!"), arena()))); EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .EndsWith(StringValue(absl::Cord("to not be stored inline!")))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .EndsWith(StringValue::WrapUnsafe("to not be stored inline!"))); EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .EndsWith(StringValue("to not be stored inline!"))); - EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .EndsWith(StringValue(absl::Cord("to not be stored inline!")))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .EndsWith(StringValue::From(absl::Cord("to not be stored inline!"), + arena()))); } TEST_F(StringValueTest, Contains) { + EXPECT_TRUE(StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .Contains(StringValue::WrapUnsafe("string is large enough"))); + EXPECT_TRUE(StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") + .Contains(StringValue::From( + absl::Cord("string is large enough"), arena()))); EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .Contains(StringValue("string is large enough"))); - EXPECT_TRUE( - StringValue("This string is large enough to not be stored inline!") - .Contains(StringValue(absl::Cord("string is large enough")))); - EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .Contains(StringValue("string is large enough"))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .Contains(StringValue::WrapUnsafe("string is large enough"))); EXPECT_TRUE( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) - .Contains(StringValue(absl::Cord("string is large enough")))); + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) + .Contains(StringValue::From(absl::Cord("string is large enough"), + arena()))); } TEST_F(StringValueTest, IndexOf) { - StringValue big_string = - StringValue("This string is large enough to not be stored inline!"); - StringValue big_string_cord = StringValue( - absl::Cord("This string is large enough to not be stored inline!")); - StringValue small_string = StringValue("is"); - StringValue small_string_cord = StringValue(absl::Cord("is")); + StringValue big_string = StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!"); + StringValue big_string_cord = StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()); + StringValue small_string = StringValue::WrapUnsafe("is"); + StringValue small_string_cord = StringValue::From(absl::Cord("is"), arena()); EXPECT_THAT(big_string.IndexOf(small_string), Optional(Eq(2))); EXPECT_THAT(big_string.IndexOf(small_string_cord), Optional(Eq(2))); @@ -249,66 +276,81 @@ TEST_F(StringValueTest, IndexOf) { } TEST_F(StringValueTest, LowerAscii) { - EXPECT_EQ(StringValue("UPPER lower").LowerAscii(arena()), "upper lower"); - EXPECT_EQ(StringValue(absl::Cord("UPPER lower")).LowerAscii(arena()), + EXPECT_EQ(StringValue::WrapUnsafe("UPPER lower").LowerAscii(arena()), "upper lower"); - EXPECT_EQ(StringValue("upper lower").LowerAscii(arena()), "upper lower"); - EXPECT_EQ(StringValue(absl::Cord("upper lower")).LowerAscii(arena()), + EXPECT_EQ( + StringValue::From(absl::Cord("UPPER lower"), arena()).LowerAscii(arena()), + "upper lower"); + EXPECT_EQ(StringValue::WrapUnsafe("upper lower").LowerAscii(arena()), "upper lower"); - EXPECT_EQ(StringValue("").LowerAscii(arena()), ""); - EXPECT_EQ(StringValue(absl::Cord("")).LowerAscii(arena()), ""); + EXPECT_EQ( + StringValue::From(absl::Cord("upper lower"), arena()).LowerAscii(arena()), + "upper lower"); + EXPECT_EQ(StringValue::WrapUnsafe("").LowerAscii(arena()), ""); + EXPECT_EQ(StringValue::From(absl::Cord(""), arena()).LowerAscii(arena()), ""); const std::string kLongMixed = "A long STRING with MiXeD case to test conversion to lower case!"; const std::string kLongLower = "a long string with mixed case to test conversion to lower case!"; - EXPECT_EQ(StringValue(absl::Cord(kLongMixed)).LowerAscii(arena()), - kLongLower); + EXPECT_EQ( + StringValue::From(absl::Cord(kLongMixed), arena()).LowerAscii(arena()), + kLongLower); std::string very_long_mixed(10000, 'A'); std::string very_long_lower(10000, 'a'); + EXPECT_EQ(StringValue::From( + absl::MakeFragmentedCord({very_long_mixed.substr(0, 5000), + very_long_mixed.substr(5000)}), + arena()) + .LowerAscii(arena()), + very_long_lower); EXPECT_EQ( - StringValue(absl::MakeFragmentedCord({very_long_mixed.substr(0, 5000), - very_long_mixed.substr(5000)})) + StringValue::From(absl::MakeFragmentedCord({"hello", "WORLD"}), arena()) .LowerAscii(arena()), - very_long_lower); - EXPECT_EQ(StringValue(absl::MakeFragmentedCord({"hello", "WORLD"})) - .LowerAscii(arena()), - "helloworld"); + "helloworld"); } TEST_F(StringValueTest, UpperAscii) { - EXPECT_EQ(StringValue("UPPER lower").UpperAscii(arena()), "UPPER LOWER"); - EXPECT_EQ(StringValue(absl::Cord("UPPER lower")).UpperAscii(arena()), + EXPECT_EQ(StringValue::WrapUnsafe("UPPER lower").UpperAscii(arena()), "UPPER LOWER"); - EXPECT_EQ(StringValue("UPPER LOWER").UpperAscii(arena()), "UPPER LOWER"); - EXPECT_EQ(StringValue(absl::Cord("UPPER LOWER")).UpperAscii(arena()), + EXPECT_EQ( + StringValue::From(absl::Cord("UPPER lower"), arena()).UpperAscii(arena()), + "UPPER LOWER"); + EXPECT_EQ(StringValue::WrapUnsafe("UPPER LOWER").UpperAscii(arena()), "UPPER LOWER"); - EXPECT_EQ(StringValue("").UpperAscii(arena()), ""); - EXPECT_EQ(StringValue(absl::Cord("")).UpperAscii(arena()), ""); + EXPECT_EQ( + StringValue::From(absl::Cord("UPPER LOWER"), arena()).UpperAscii(arena()), + "UPPER LOWER"); + EXPECT_EQ(StringValue::WrapUnsafe("").UpperAscii(arena()), ""); + EXPECT_EQ(StringValue::From(absl::Cord(""), arena()).UpperAscii(arena()), ""); const std::string kLongMixed = "A long STRING with MiXeD case to test conversion to UPPER case!"; const std::string kLongUpper = "A LONG STRING WITH MIXED CASE TO TEST CONVERSION TO UPPER CASE!"; - EXPECT_EQ(StringValue(absl::Cord(kLongMixed)).UpperAscii(arena()), - kLongUpper); + EXPECT_EQ( + StringValue::From(absl::Cord(kLongMixed), arena()).UpperAscii(arena()), + kLongUpper); std::string very_long_mixed(10000, 'a'); std::string very_long_upper(10000, 'A'); + EXPECT_EQ(StringValue::From( + absl::MakeFragmentedCord({very_long_mixed.substr(0, 5000), + very_long_mixed.substr(5000)}), + arena()) + .UpperAscii(arena()), + very_long_upper); EXPECT_EQ( - StringValue(absl::MakeFragmentedCord({very_long_mixed.substr(0, 5000), - very_long_mixed.substr(5000)})) + StringValue::From(absl::MakeFragmentedCord({"HELLO", "world"}), arena()) .UpperAscii(arena()), - very_long_upper); - EXPECT_EQ(StringValue(absl::MakeFragmentedCord({"HELLO", "world"})) - .UpperAscii(arena()), - "HELLOWORLD"); + "HELLOWORLD"); } TEST_F(StringValueTest, LastIndexOf) { - StringValue big_string = - StringValue("This string is large enough to not be stored inline!"); - StringValue big_string_cord = StringValue( - absl::Cord("This string is large enough to not be stored inline!")); - StringValue small_string = StringValue("is"); - StringValue small_string_cord = StringValue(absl::Cord("is")); + StringValue big_string = StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!"); + StringValue big_string_cord = StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()); + StringValue small_string = StringValue::WrapUnsafe("is"); + StringValue small_string_cord = StringValue::From(absl::Cord("is"), arena()); EXPECT_THAT(big_string.LastIndexOf(small_string), Optional(Eq(12))); EXPECT_THAT(big_string.LastIndexOf(small_string_cord), Optional(Eq(12))); @@ -346,12 +388,12 @@ TEST_F(StringValueTest, LastIndexOf) { TEST_F(StringValueTest, Trim) { using ::cel::test::StringValueIs; - StringValue unpadded = StringValue("no padding"); - StringValue front_padded = StringValue(" \t\r\nno padding"); - StringValue back_padded = StringValue("no padding \t\r\n"); - StringValue both_padded = StringValue(" \t\r\nno padding \t\r\n"); - StringValue whitespace = StringValue(" \t\r\n"); - StringValue empty = StringValue(""); + StringValue unpadded = StringValue::WrapUnsafe("no padding"); + StringValue front_padded = StringValue::WrapUnsafe(" \t\r\nno padding"); + StringValue back_padded = StringValue::WrapUnsafe("no padding \t\r\n"); + StringValue both_padded = StringValue::WrapUnsafe(" \t\r\nno padding \t\r\n"); + StringValue whitespace = StringValue::WrapUnsafe(" \t\r\n"); + StringValue empty = StringValue::WrapUnsafe(""); EXPECT_THAT(unpadded.Trim(), StringValueIs("no padding")); EXPECT_THAT(front_padded.Trim(), StringValueIs("no padding")); @@ -360,13 +402,17 @@ TEST_F(StringValueTest, Trim) { EXPECT_THAT(whitespace.Trim(), StringValueIs("")); EXPECT_THAT(empty.Trim(), StringValueIs("")); - StringValue unpadded_cord = StringValue(absl::Cord("no padding")); - StringValue front_padded_cord = StringValue(absl::Cord(" \t\r\nno padding")); - StringValue back_padded_cord = StringValue(absl::Cord("no padding \t\r\n")); + StringValue unpadded_cord = + StringValue::From(absl::Cord("no padding"), arena()); + StringValue front_padded_cord = + StringValue::From(absl::Cord(" \t\r\nno padding"), arena()); + StringValue back_padded_cord = + StringValue::From(absl::Cord("no padding \t\r\n"), arena()); StringValue both_padded_cord = - StringValue(absl::Cord(" \t\r\nno padding \t\r\n")); - StringValue whitespace_cord = StringValue(absl::Cord(" \t\r\n")); - StringValue empty_cord = StringValue(absl::Cord("")); + StringValue::From(absl::Cord(" \t\r\nno padding \t\r\n"), arena()); + StringValue whitespace_cord = + StringValue::From(absl::Cord(" \t\r\n"), arena()); + StringValue empty_cord = StringValue::From(absl::Cord(""), arena()); EXPECT_THAT(unpadded_cord.Trim(), StringValueIs("no padding")); EXPECT_THAT(front_padded_cord.Trim(), StringValueIs("no padding")); @@ -379,14 +425,16 @@ TEST_F(StringValueTest, Trim) { TEST_F(StringValueTest, CharAt) { using ::cel::test::ErrorValueIs; using ::cel::test::StringValueIs; - StringValue big_string = - StringValue("This string is large enough to not be stored inline!"); - StringValue big_string_cord = StringValue( - absl::Cord("This string is large enough to not be stored inline!")); - StringValue small_string = StringValue("abc"); - StringValue small_string_cord = StringValue(absl::Cord("abc")); - StringValue unicode_string = StringValue("aμc"); - StringValue unicode_string_cord = StringValue(absl::Cord("aμc")); + StringValue big_string = StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!"); + StringValue big_string_cord = StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()); + StringValue small_string = StringValue::WrapUnsafe("abc"); + StringValue small_string_cord = StringValue::From(absl::Cord("abc"), arena()); + StringValue unicode_string = StringValue::WrapUnsafe("aμc"); + StringValue unicode_string_cord = + StringValue::From(absl::Cord("aμc"), arena()); EXPECT_THAT(big_string.CharAt(0), StringValueIs("T")); EXPECT_THAT(big_string_cord.CharAt(0), StringValueIs("T")); @@ -419,8 +467,8 @@ TEST_F(StringValueTest, Substring) { // as a large (cord-backed) value. The substring length must be measured in // code units, not code points, otherwise the cord overload truncates a // multi-byte character or underflows the length. - StringValue unicode_cord = StringValue(absl::Cord("€€€€€€")); - StringValue unicode_view = StringValue("€€€€€€"); + StringValue unicode_cord = StringValue::From(absl::Cord("€€€€€€"), arena()); + StringValue unicode_view = StringValue::WrapUnsafe("€€€€€€"); EXPECT_THAT(unicode_cord.Substring(0, 2), StringValueIs("€€")); EXPECT_THAT(unicode_view.Substring(0, 2), StringValueIs("€€")); @@ -445,7 +493,7 @@ TEST_F(StringValueTest, Join) { using ::cel::test::ErrorValueIs; using ::cel::test::StringValueIs; - StringValue separator(","); + StringValue separator = StringValue::WrapUnsafe(","); Value result; // Empty list. @@ -458,7 +506,7 @@ TEST_F(StringValueTest, Join) { // Single element list. auto list_builder1 = NewListValueBuilder(arena()); - ASSERT_THAT(list_builder1->Add(StringValue("foo")), IsOk()); + ASSERT_THAT(list_builder1->Add(StringValue::WrapUnsafe("foo")), IsOk()); auto list1 = std::move(*list_builder1).Build(); EXPECT_THAT(separator.Join(list1, descriptor_pool(), message_factory(), arena(), &result), @@ -467,9 +515,9 @@ TEST_F(StringValueTest, Join) { // Multi element list. auto list_builder2 = NewListValueBuilder(arena()); - ASSERT_THAT(list_builder2->Add(StringValue("foo")), IsOk()); - ASSERT_THAT(list_builder2->Add(StringValue("bar")), IsOk()); - ASSERT_THAT(list_builder2->Add(StringValue("baz")), IsOk()); + ASSERT_THAT(list_builder2->Add(StringValue::WrapUnsafe("foo")), IsOk()); + ASSERT_THAT(list_builder2->Add(StringValue::WrapUnsafe("bar")), IsOk()); + ASSERT_THAT(list_builder2->Add(StringValue::WrapUnsafe("baz")), IsOk()); auto list2 = std::move(*list_builder2).Build(); EXPECT_THAT(separator.Join(list2, descriptor_pool(), message_factory(), arena(), &result), @@ -487,7 +535,7 @@ TEST_F(StringValueTest, Join) { // List with string and non-string. auto list_builder4 = NewListValueBuilder(arena()); - ASSERT_THAT(list_builder4->Add(StringValue("foo")), IsOk()); + ASSERT_THAT(list_builder4->Add(StringValue::WrapUnsafe("foo")), IsOk()); ASSERT_THAT(list_builder4->Add(IntValue(1)), IsOk()); auto list4 = std::move(*list_builder4).Build(); EXPECT_THAT(separator.Join(list4, descriptor_pool(), message_factory(), @@ -500,20 +548,24 @@ TEST_F(StringValueTest, Reverse) { using ::cel::test::StringValueIs; EXPECT_THAT(StringValue().Reverse(arena()), StringValueIs("")); - EXPECT_THAT(StringValue("").Reverse(arena()), StringValueIs("")); - EXPECT_THAT(StringValue("hello").Reverse(arena()), StringValueIs("olleh")); - EXPECT_THAT(StringValue("aμc").Reverse(arena()), StringValueIs("cμa")); + EXPECT_THAT(StringValue::WrapUnsafe("").Reverse(arena()), StringValueIs("")); + EXPECT_THAT(StringValue::WrapUnsafe("hello").Reverse(arena()), + StringValueIs("olleh")); + EXPECT_THAT(StringValue::WrapUnsafe("aμc").Reverse(arena()), + StringValueIs("cμa")); EXPECT_THAT( - StringValue("This string is large enough to not be stored inline!") + StringValue::WrapUnsafe( + "This string is large enough to not be stored inline!") .Reverse(arena()), StringValueIs("!enilni derots eb ton ot hguone egral si gnirts sihT")); - EXPECT_THAT(StringValue(absl::Cord("hello")).Reverse(arena()), + EXPECT_THAT(StringValue::From(absl::Cord("hello"), arena()).Reverse(arena()), StringValueIs("olleh")); - EXPECT_THAT(StringValue(absl::Cord("aμc")).Reverse(arena()), + EXPECT_THAT(StringValue::From(absl::Cord("aμc"), arena()).Reverse(arena()), StringValueIs("cμa")); EXPECT_THAT( - StringValue( - absl::Cord("This string is large enough to not be stored inline!")) + StringValue::From( + absl::Cord("This string is large enough to not be stored inline!"), + arena()) .Reverse(arena()), StringValueIs("!enilni derots eb ton ot hguone egral si gnirts sihT")); } diff --git a/common/values/value_variant.cc b/common/values/value_variant.cc index 1c287239c..7c9981d83 100644 --- a/common/values/value_variant.cc +++ b/common/values/value_variant.cc @@ -21,9 +21,7 @@ #include "absl/base/optimization.h" #include "absl/log/absl_check.h" -#include "common/values/bytes_value.h" #include "common/values/error_value.h" -#include "common/values/string_value.h" #include "common/values/unknown_value.h" #include "common/values/values.h" @@ -33,13 +31,6 @@ void ValueVariant::SlowCopyConstruct(const ValueVariant& other) noexcept { ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); switch (index_) { - case ValueIndex::kBytes: - ::new (static_cast(&raw_[0])) BytesValue(*other.At()); - break; - case ValueIndex::kString: - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - break; case ValueIndex::kError: ::new (static_cast(&raw_[0])) ErrorValue(*other.At()); break; @@ -56,14 +47,6 @@ void ValueVariant::SlowMoveConstruct(ValueVariant& other) noexcept { ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); switch (index_) { - case ValueIndex::kBytes: - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - break; - case ValueIndex::kString: - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - break; case ValueIndex::kError: ::new (static_cast(&raw_[0])) ErrorValue(std::move(*other.At())); @@ -81,12 +64,6 @@ void ValueVariant::SlowDestruct() noexcept { ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); switch (index_) { - case ValueIndex::kBytes: - At()->~BytesValue(); - break; - case ValueIndex::kString: - At()->~StringValue(); - break; case ValueIndex::kError: At()->~ErrorValue(); break; @@ -104,14 +81,6 @@ void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, if (trivial) { switch (other.index_) { - case ValueIndex::kBytes: - ::new (static_cast(&raw_[0])) - BytesValue(*other.At()); - break; - case ValueIndex::kString: - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - break; case ValueIndex::kError: ::new (static_cast(&raw_[0])) ErrorValue(*other.At()); @@ -128,12 +97,6 @@ void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, flags_ = other.flags_; } else if (other_trivial) { switch (index_) { - case ValueIndex::kBytes: - At()->~BytesValue(); - break; - case ValueIndex::kString: - At()->~StringValue(); - break; case ValueIndex::kError: At()->~ErrorValue(); break; @@ -146,82 +109,8 @@ void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, FastCopyAssign(other); } else { switch (index_) { - case ValueIndex::kBytes: - switch (other.index_) { - case ValueIndex::kBytes: - *At() = *other.At(); - break; - case ValueIndex::kString: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kError: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - case ValueIndex::kString: - switch (other.index_) { - case ValueIndex::kBytes: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - BytesValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - *At() = *other.At(); - break; - case ValueIndex::kError: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; case ValueIndex::kError: switch (other.index_) { - case ValueIndex::kBytes: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - BytesValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; case ValueIndex::kError: *At() = *other.At(); break; @@ -238,20 +127,6 @@ void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, break; case ValueIndex::kUnknown: switch (other.index_) { - case ValueIndex::kBytes: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - BytesValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - StringValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; case ValueIndex::kError: At()->~UnknownValue(); ::new (static_cast(&raw_[0])) @@ -283,14 +158,6 @@ void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, if (trivial) { switch (other.index_) { - case ValueIndex::kBytes: - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - break; - case ValueIndex::kString: - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - break; case ValueIndex::kError: ::new (static_cast(&raw_[0])) ErrorValue(std::move(*other.At())); @@ -307,12 +174,6 @@ void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, flags_ = other.flags_; } else if (other_trivial) { switch (index_) { - case ValueIndex::kBytes: - At()->~BytesValue(); - break; - case ValueIndex::kString: - At()->~StringValue(); - break; case ValueIndex::kError: At()->~ErrorValue(); break; @@ -325,82 +186,8 @@ void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, FastMoveAssign(other); } else { switch (index_) { - case ValueIndex::kBytes: - switch (other.index_) { - case ValueIndex::kBytes: - *At() = std::move(*other.At()); - break; - case ValueIndex::kString: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kError: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~BytesValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - case ValueIndex::kString: - switch (other.index_) { - case ValueIndex::kBytes: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - *At() = std::move(*other.At()); - break; - case ValueIndex::kError: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kUnknown: - At()->~StringValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; case ValueIndex::kError: switch (other.index_) { - case ValueIndex::kBytes: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; case ValueIndex::kError: *At() = std::move(*other.At()); break; @@ -417,20 +204,6 @@ void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, break; case ValueIndex::kUnknown: switch (other.index_) { - case ValueIndex::kBytes: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - BytesValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - case ValueIndex::kString: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - StringValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; case ValueIndex::kError: At()->~UnknownValue(); ::new (static_cast(&raw_[0])) @@ -463,16 +236,6 @@ void ValueVariant::SlowSwap(ValueVariant& lhs, ValueVariant& rhs, // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) std::memcpy(tmp, std::addressof(lhs), sizeof(ValueVariant)); switch (rhs.index_) { - case ValueIndex::kBytes: - ::new (static_cast(&lhs.raw_[0])) - BytesValue(*rhs.At()); - rhs.At()->~BytesValue(); - break; - case ValueIndex::kString: - ::new (static_cast(&lhs.raw_[0])) - StringValue(*rhs.At()); - rhs.At()->~StringValue(); - break; case ValueIndex::kError: ::new (static_cast(&lhs.raw_[0])) ErrorValue(*rhs.At()); @@ -498,16 +261,6 @@ void ValueVariant::SlowSwap(ValueVariant& lhs, ValueVariant& rhs, // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) std::memcpy(tmp, std::addressof(rhs), sizeof(ValueVariant)); switch (lhs.index_) { - case ValueIndex::kBytes: - ::new (static_cast(&rhs.raw_[0])) - BytesValue(*lhs.At()); - lhs.At()->~BytesValue(); - break; - case ValueIndex::kString: - ::new (static_cast(&rhs.raw_[0])) - StringValue(*lhs.At()); - lhs.At()->~StringValue(); - break; case ValueIndex::kError: ::new (static_cast(&rhs.raw_[0])) ErrorValue(*lhs.At()); diff --git a/common/values/value_variant.h b/common/values/value_variant.h index b05511e3c..5a26a742f 100644 --- a/common/values/value_variant.h +++ b/common/values/value_variant.h @@ -86,10 +86,9 @@ enum class ValueIndex : uint8_t { kParsedMessage, kCustomStruct, kOpaque, - - // Keep non-trivial alternatives together to aid in compiling optimizations. kBytes, kString, + // Keep non-trivial alternatives together to aid in compiling optimizations. kError, kUnknown, }; @@ -355,9 +354,7 @@ struct ValueAlternative { static constexpr bool kAlwaysTrivial = false; static ValueFlags Flags(const BytesValue* absl_nonnull alternative) { - return ArenaTraits::trivially_destructible(*alternative) - ? ValueFlags::kNone - : ValueFlags::kNonTrivial; + return ValueFlags::kNone; } }; @@ -368,9 +365,7 @@ struct ValueAlternative { static constexpr bool kAlwaysTrivial = false; static ValueFlags Flags(const StringValue* absl_nonnull alternative) { - return ArenaTraits::trivially_destructible(*alternative) - ? ValueFlags::kNone - : ValueFlags::kNonTrivial; + return ValueFlags::kNone; } }; diff --git a/common/values/value_variant_test.cc b/common/values/value_variant_test.cc index 1fd3629aa..968dca0d9 100644 --- a/common/values/value_variant_test.cc +++ b/common/values/value_variant_test.cc @@ -63,16 +63,16 @@ struct DefaultValue { template <> struct DefaultValue { BytesValue operator()() const { - return BytesValue( - absl::Cord("Some somewhat large string that is not storable inline!")); + return BytesValue::WrapUnsafe( + "Some somewhat large string that is not storable inline!"); } }; template <> struct DefaultValue { StringValue operator()() const { - return StringValue( - absl::Cord("Some somewhat large string that is not storable inline!")); + return StringValue::WrapUnsafe( + "Some somewhat large string that is not storable inline!"); } }; diff --git a/conformance/policy/policy_conformance_test.cc b/conformance/policy/policy_conformance_test.cc index 53fbcb419..c325dace8 100644 --- a/conformance/policy/policy_conformance_test.cc +++ b/conformance/policy/policy_conformance_test.cc @@ -136,9 +136,9 @@ cel::Value LocationCode(const cel::StringValue& ip, const google::protobuf::DescriptorPool* pool, google::protobuf::MessageFactory* factory, google::protobuf::Arena* arena) { std::string ip_str = ip.ToString(); - if (ip_str == "10.0.0.1") return cel::StringValue(arena, "us"); - if (ip_str == "10.0.0.2") return cel::StringValue(arena, "de"); - return cel::StringValue(arena, "ir"); + if (ip_str == "10.0.0.1") return cel::StringValue::WrapUnsafe("us"); + if (ip_str == "10.0.0.2") return cel::StringValue::WrapUnsafe("de"); + return cel::StringValue::WrapUnsafe("ir"); } // TODO(uncreated-issue/92): This should be migrated to use the testrunner utility @@ -209,7 +209,9 @@ class InputEvaluator { cel::Activation activation; EvaluateOptions options; options.message_factory = message_factory; - return program->Evaluate(arena, activation, options); + CEL_ASSIGN_OR_RETURN(auto result, + program->Evaluate(arena, activation, options)); + return result.Clone(arena); } private: diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 53f7cf0c4..5a2490976 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -716,7 +716,7 @@ class FlatExprVisitor : public cel::AstVisitor { } absl::StatusOr converted_value = - ConvertConstant(const_expr, cel::NewDeleteAllocator()); + ConvertConstant(const_expr, extension_context_.MutableArena()); if (!converted_value.ok()) { SetProgressStatusIfError(converted_value.status()); @@ -995,7 +995,7 @@ class FlatExprVisitor : public cel::AstVisitor { return; } - StringValue field = cel::StringValue(select_expr.field()); + std::string field = select_expr.field(); std::optional struct_type; std::optional field_type; if (options_.enable_typed_field_access) { diff --git a/eval/eval/comprehension_slots_test.cc b/eval/eval/comprehension_slots_test.cc index 5f869d7cb..3dba2f80e 100644 --- a/eval/eval/comprehension_slots_test.cc +++ b/eval/eval/comprehension_slots_test.cc @@ -38,7 +38,7 @@ TEST(ComprehensionSlots, Basic) { ComprehensionSlots::Slot* slot0 = slots.Get(0); EXPECT_FALSE(slot0->Has()); - slots.Set(0, cel::StringValue("abcd"), + slots.Set(0, cel::StringValue::WrapUnsafe("abcd"), AttributeTrail(Attribute("fake_attr"))); ASSERT_TRUE(slot0->Has()); @@ -55,7 +55,7 @@ TEST(ComprehensionSlots, Basic) { slots.ClearSlot(0); EXPECT_FALSE(slot0->Has()); - slots.Set(3, cel::StringValue("abcd"), + slots.Set(3, cel::StringValue::WrapUnsafe("abcd"), AttributeTrail(Attribute("fake_attr"))); auto* slot3 = slots.Get(3); diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index 3d3bae34d..1bc6b35ee 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -1186,7 +1186,7 @@ TEST_F(DirectFunctionStepTest, NoOverload) { std::vector> deps; deps.push_back(CreateConstValueDirectStep(cel::IntValue(1))); - deps.push_back(CreateConstValueDirectStep(cel::StringValue("2"))); + deps.push_back(CreateConstValueDirectStep(cel::StringValue::WrapUnsafe("2"))); auto expr = CreateDirectFunctionStep(-1, call, std::move(deps), GetOverloads(cel::builtin::kAdd, 2)); diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 67054d75d..ca33a774f 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -206,11 +206,10 @@ absl::Status PerformOptionalGet(const Value& target, absl::string_view field, // message. class SelectStep : public ExpressionStepBase { public: - SelectStep(StringValue value, bool test_field_presence, int64_t expr_id, + SelectStep(absl::string_view field, bool test_field_presence, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) : ExpressionStepBase(expr_id), - field_value_(std::move(value)), - field_(field_value_.ToString()), + field_(field), unboxing_option_(enable_wrapper_type_null_unboxing ? ProtoWrapperTypeOptions::kUnsetNull : ProtoWrapperTypeOptions::kUnsetProtoDefault), @@ -220,7 +219,6 @@ class SelectStep : public ExpressionStepBase { absl::Status Evaluate(ExecutionFrame* frame) const override; protected: - cel::StringValue field_value_; std::string field_; ProtoWrapperTypeOptions unboxing_option_; bool test_field_presence_; @@ -282,8 +280,9 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { target = &result; } CEL_RETURN_IF_ERROR( - PerformHas(*target, field_, field_value_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), result)); + PerformHas(*target, field_, cel::StringValue::WrapUnsafe(field_), + frame->descriptor_pool(), frame->message_factory(), + frame->arena(), result)); frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); return absl::OkStatus(); } @@ -297,8 +296,8 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { Value value; optional_arg->Value(&value); auto status = PerformOptionalGet( - value, field_, field_value_, unboxing_option_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), + value, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, + frame->descriptor_pool(), frame->message_factory(), frame->arena(), frame->options().enable_use_new_field_select_implementation, result); if (!status.ok()) { result = ErrorValue::From(std::move(status), frame->arena()); @@ -308,8 +307,8 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { } CEL_RETURN_IF_ERROR(PerformGet( - arg, field_, field_value_, unboxing_option_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), + arg, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, + frame->descriptor_pool(), frame->message_factory(), frame->arena(), frame->options().enable_use_new_field_select_implementation, result)); frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); return absl::OkStatus(); @@ -319,13 +318,12 @@ class DirectSelectStep : public DirectExpressionStep { public: DirectSelectStep(int64_t expr_id, std::unique_ptr operand, - StringValue field, bool test_only, + absl::string_view field, bool test_only, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) : DirectExpressionStep(expr_id), operand_(std::move(operand)), - field_value_(std::move(field)), - field_(field_value_.ToString()), + field_(field), test_only_(test_only), unboxing_option_(enable_wrapper_type_null_unboxing ? ProtoWrapperTypeOptions::kUnsetNull @@ -377,11 +375,13 @@ class DirectSelectStep : public DirectExpressionStep { } Value value; optional_arg->Value(&value); - return PerformHas(value, field_, field_value_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), result); + return PerformHas(value, field_, cel::StringValue::WrapUnsafe(field_), + frame.descriptor_pool(), frame.message_factory(), + frame.arena(), result); } - return PerformHas(result, field_, field_value_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), result); + return PerformHas(result, field_, cel::StringValue::WrapUnsafe(field_), + frame.descriptor_pool(), frame.message_factory(), + frame.arena(), result); } if (optional_arg) { @@ -392,7 +392,7 @@ class DirectSelectStep : public DirectExpressionStep { Value value; optional_arg->Value(&value); auto status = PerformOptionalGet( - value, field_, field_value_, unboxing_option_, + value, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, frame.descriptor_pool(), frame.message_factory(), frame.arena(), frame.options().enable_use_new_field_select_implementation, result); if (!status.ok()) { @@ -402,8 +402,8 @@ class DirectSelectStep : public DirectExpressionStep { } return PerformGet( - result, field_, field_value_, unboxing_option_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), + result, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, + frame.descriptor_pool(), frame.message_factory(), frame.arena(), frame.options().enable_use_new_field_select_implementation, result); } @@ -415,7 +415,6 @@ class DirectSelectStep : public DirectExpressionStep { // // ToString or ValueManager::CreateString may force a copy so we do this at // plan time. - StringValue field_value_; std::string field_; // whether this is a has() expression. @@ -462,12 +461,12 @@ bool SupportsCachedFieldDescriptor( class ProtoSelectStep : public SelectStep { public: - ProtoSelectStep(StringValue value, int64_t expr_id, + ProtoSelectStep(absl::string_view value, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types, const google::protobuf::Descriptor* descriptor, const google::protobuf::FieldDescriptor* field_descriptor) - : SelectStep(std::move(value), /*test_field_presence=*/false, expr_id, + : SelectStep(value, /*test_field_presence=*/false, expr_id, enable_wrapper_type_null_unboxing, enable_optional_types), descriptor_(descriptor), field_descriptor_(field_descriptor) { @@ -542,11 +541,11 @@ absl::Status ProtoSelectStep::EvaluateMessageFieldGet( class ProtoHasStep : public SelectStep { public: - ProtoHasStep(StringValue value, int64_t expr_id, + ProtoHasStep(absl::string_view field, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types, const google::protobuf::Descriptor* descriptor, const google::protobuf::FieldDescriptor* field_descriptor) - : SelectStep(std::move(value), /*test_field_presence=*/true, expr_id, + : SelectStep(field, /*test_field_presence=*/true, expr_id, enable_wrapper_type_null_unboxing, enable_optional_types), descriptor_(descriptor), field_descriptor_(field_descriptor) { @@ -603,7 +602,7 @@ absl::Status ProtoHasStep::EvaluateHas( } // namespace std::unique_ptr CreateDirectSelectStep( - std::unique_ptr operand, StringValue field, + std::unique_ptr operand, absl::string_view field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { return std::make_unique( @@ -613,7 +612,7 @@ std::unique_ptr CreateDirectSelectStep( // Factory method for Select - based Execution step absl::StatusOr> CreateSelectStep( - cel::StringValue field, bool test_only, int64_t expr_id, + absl::string_view field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { return std::make_unique(std::move(field), test_only, expr_id, enable_wrapper_type_null_unboxing, @@ -622,7 +621,7 @@ absl::StatusOr> CreateSelectStep( // Factory method for Select - based Execution step absl::StatusOr> CreateTypedSelectStep( - cel::StringValue field, cel::StructType resolved_operand_type, + absl::string_view field, cel::StructType resolved_operand_type, cel::StructTypeField resolved_field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { if (!resolved_operand_type.IsMessage()) { diff --git a/eval/eval/select_step.h b/eval/eval/select_step.h index c3f965a94..17a93d19f 100644 --- a/eval/eval/select_step.h +++ b/eval/eval/select_step.h @@ -5,8 +5,8 @@ #include #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "common/type.h" -#include "common/value.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" @@ -14,17 +14,17 @@ namespace google::api::expr::runtime { // Factory method for recursively evaluated select step. std::unique_ptr CreateDirectSelectStep( - std::unique_ptr operand, cel::StringValue field, + std::unique_ptr operand, absl::string_view field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types = false); // Factory method for Select stack machine based Execution step absl::StatusOr> CreateSelectStep( - cel::StringValue field, bool test_only, int64_t expr_id, + absl::string_view field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_ytpes = false); absl::StatusOr> CreateTypedSelectStep( - cel::StringValue field, cel::StructType resolved_operand_type, + absl::string_view field, cel::StructType resolved_operand_type, cel::StructTypeField resolved_field, bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, bool enable_optional_types); diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index f19370cdc..b9925fb52 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -107,8 +107,8 @@ class SelectStepTest : public testing::Test { CEL_ASSIGN_OR_RETURN(auto step0, CreateIdentStep(ident.name(), expr0.id())); CEL_ASSIGN_OR_RETURN( auto step1, - CreateSelectStep(cel::StringValue(select.field()), select.test_only(), - expr.id(), options.enable_wrapper_type_null_unboxing)); + CreateSelectStep(select.field(), select.test_only(), expr.id(), + options.enable_wrapper_type_null_unboxing)); path.push_back(std::move(step0)); path.push_back(std::move(step1)); @@ -290,13 +290,11 @@ TEST_F(SelectStepTest, MapPresenseIsErrorTest) { ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(cel::StringValue(select_map.field()), - select_map.test_only(), expr1.id(), + CreateSelectStep(select_map.field(), select_map.test_only(), expr1.id(), /*enable_wrapper_type_null_unboxing=*/false)); ASSERT_OK_AND_ASSIGN( auto step2, - CreateSelectStep(cel::StringValue(select.field()), select.test_only(), - select_expr.id(), + CreateSelectStep(select.field(), select.test_only(), select_expr.id(), /*enable_wrapper_type_null_unboxing=*/false)); ExecutionPath path; @@ -752,8 +750,7 @@ TEST_P(SelectStepConformanceTest, CelErrorAsArgument) { ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(cel::StringValue(select.field()), select.test_only(), - dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), dummy_expr.id(), /*enable_wrapper_type_null_unboxing=*/false)); path.push_back(std::move(step0)); @@ -794,8 +791,7 @@ TEST_F(SelectStepTest, DisableMissingAttributeOK) { ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(cel::StringValue(select.field()), select.test_only(), - dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), dummy_expr.id(), /*enable_wrapper_type_null_unboxing=*/false)); path.push_back(std::move(step0)); @@ -837,8 +833,7 @@ TEST_F(SelectStepTest, UnrecoverableUnknownValueProducesError) { ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(cel::StringValue(select.field()), select.test_only(), - dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), dummy_expr.id(), /*enable_wrapper_type_null_unboxing=*/false)); path.push_back(std::move(step0)); @@ -884,9 +879,9 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); auto step0_status = CreateIdentStep(ident.name(), expr0.id()); - auto step1_status = CreateSelectStep( - cel::StringValue(select.field()), select.test_only(), dummy_expr.id(), - /*enable_wrapper_type_null_unboxing=*/false); + auto step1_status = + CreateSelectStep(select.field(), select.test_only(), dummy_expr.id(), + /*enable_wrapper_type_null_unboxing=*/false); ASSERT_THAT(step0_status, IsOk()); ASSERT_THAT(step1_status, IsOk()); @@ -987,12 +982,11 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); ASSERT_OK_AND_ASSIGN( - auto step1, - CreateTypedSelectStep(cel::StringValue("single_int64"), - resolved_operand_type, resolved_field, - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/false, - /*enable_optional_types=*/false)); + auto step1, CreateTypedSelectStep( + "single_int64", resolved_operand_type, resolved_field, + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); ExecutionPath path; path.push_back(std::move(step0)); @@ -1028,12 +1022,11 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); ASSERT_OK_AND_ASSIGN( - auto step1, - CreateTypedSelectStep(cel::StringValue("single_int64"), - resolved_operand_type, resolved_field, - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/false, - /*enable_optional_types=*/false)); + auto step1, CreateTypedSelectStep( + "single_int64", resolved_operand_type, resolved_field, + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); ExecutionPath path; path.push_back(std::move(step0)); @@ -1066,12 +1059,11 @@ TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { cel::StructTypeField resolved_field((cel::MessageTypeField(field_desc))); ASSERT_OK_AND_ASSIGN( - auto step1, - CreateTypedSelectStep(cel::StringValue("single_int64"), - resolved_operand_type, resolved_field, - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/false, - /*enable_optional_types=*/false)); + auto step1, CreateTypedSelectStep( + "single_int64", resolved_operand_type, resolved_field, + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/false, + /*enable_optional_types=*/false)); ExecutionPath path; path.push_back(std::move(step0)); @@ -1133,14 +1125,18 @@ TEST_F(DirectSelectStepTest, SelectFromMap) { cel::Activation activation; RuntimeOptions options; - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("map_val", -1), cel::StringValue("one"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); + auto step = + CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "one", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true); auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), + IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), + IsOk()); activation.InsertOrAssignValue("map_val", std::move(*map_builder).Build()); ExecutionFrameBase frame(activation, options, type_provider_, @@ -1160,14 +1156,18 @@ TEST_F(DirectSelectStepTest, HasMap) { cel::Activation activation; RuntimeOptions options; - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("map_val", -1), cel::StringValue("two"), - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true); + auto step = + CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "two", + /*test_only=*/true, -1, + /*enable_wrapper_type_null_unboxing=*/true); auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), + IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), + IsOk()); activation.InsertOrAssignValue("map_val", std::move(*map_builder).Build()); ExecutionFrameBase frame(activation, options, type_provider_, @@ -1187,15 +1187,19 @@ TEST_F(DirectSelectStepTest, SelectFromOptionalMap) { cel::Activation activation; RuntimeOptions options; - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("one"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); + auto step = + CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "one", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true, + /*enable_optional_types=*/true); auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), + IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), + IsOk()); activation.InsertOrAssignValue( "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); @@ -1216,15 +1220,19 @@ TEST_F(DirectSelectStepTest, SelectFromOptionalMapAbsent) { cel::Activation activation; RuntimeOptions options; - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("three"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); + auto step = + CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "three", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true, + /*enable_optional_types=*/true); auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), + IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), + IsOk()); activation.InsertOrAssignValue( "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); @@ -1246,7 +1254,7 @@ TEST_F(DirectSelectStepTest, SelectFromOptionalStruct) { RuntimeOptions options; auto step = CreateDirectSelectStep(CreateDirectIdentStep("struct_val", -1), - cel::StringValue("single_int64"), + "single_int64", /*test_only=*/false, -1, /*enable_wrapper_type_null_unboxing=*/true, /*enable_optional_types=*/true); @@ -1281,7 +1289,7 @@ TEST_F(DirectSelectStepTest, SelectFromOptionalStructFieldNotSet) { RuntimeOptions options; auto step = CreateDirectSelectStep(CreateDirectIdentStep("struct_val", -1), - cel::StringValue("single_string"), + "single_string", /*test_only=*/false, -1, /*enable_wrapper_type_null_unboxing=*/true, /*enable_optional_types=*/true); @@ -1315,11 +1323,11 @@ TEST_F(DirectSelectStepTest, SelectFromEmptyOptional) { cel::Activation activation; RuntimeOptions options; - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("one"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); + auto step = + CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "one", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true, + /*enable_optional_types=*/true); activation.InsertOrAssignValue("map_val", OptionalValue::None()); @@ -1340,15 +1348,19 @@ TEST_F(DirectSelectStepTest, HasOptional) { cel::Activation activation; RuntimeOptions options; - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("two"), - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); + auto step = + CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "two", + /*test_only=*/true, -1, + /*enable_wrapper_type_null_unboxing=*/true, + /*enable_optional_types=*/true); auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT(map_builder->Put(cel::StringValue("one"), IntValue(1)), IsOk()); - ASSERT_THAT(map_builder->Put(cel::StringValue("two"), IntValue(2)), IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), + IsOk()); + ASSERT_THAT( + map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), + IsOk()); activation.InsertOrAssignValue( "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); @@ -1369,11 +1381,11 @@ TEST_F(DirectSelectStepTest, HasEmptyOptional) { cel::Activation activation; RuntimeOptions options; - auto step = CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), - cel::StringValue("two"), - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); + auto step = + CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "two", + /*test_only=*/true, -1, + /*enable_wrapper_type_null_unboxing=*/true, + /*enable_optional_types=*/true); activation.InsertOrAssignValue("map_val", OptionalValue::None()); @@ -1394,11 +1406,10 @@ TEST_F(DirectSelectStepTest, SelectFromStruct) { cel::Activation activation; RuntimeOptions options; - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); + auto step = CreateDirectSelectStep( + CreateDirectIdentStep("test_all_types", -1), "single_int64", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true); TestAllTypes message; message.set_single_int64(1); @@ -1421,11 +1432,10 @@ TEST_F(DirectSelectStepTest, HasStruct) { cel::Activation activation; RuntimeOptions options; - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_string"), - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true); + auto step = CreateDirectSelectStep( + CreateDirectIdentStep("test_all_types", -1), "single_string", + /*test_only=*/true, -1, + /*enable_wrapper_type_null_unboxing=*/true); TestAllTypes message; message.set_single_int64(1); @@ -1449,10 +1459,10 @@ TEST_F(DirectSelectStepTest, SelectFromUnsupportedType) { cel::Activation activation; RuntimeOptions options; - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("bool_val", -1), cel::StringValue("one"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); + auto step = + CreateDirectSelectStep(CreateDirectIdentStep("bool_val", -1), "one", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true); activation.InsertOrAssignValue("bool_val", BoolValue(false)); @@ -1476,11 +1486,10 @@ TEST_F(DirectSelectStepTest, AttributeUpdatedIfRequested) { RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); + auto step = CreateDirectSelectStep( + CreateDirectIdentStep("test_all_types", -1), "single_int64", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true); TestAllTypes message; message.set_single_int64(1); @@ -1506,11 +1515,10 @@ TEST_F(DirectSelectStepTest, MissingAttributesToErrors) { RuntimeOptions options; options.enable_missing_attribute_errors = true; - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); + auto step = CreateDirectSelectStep( + CreateDirectIdentStep("test_all_types", -1), "single_int64", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true); TestAllTypes message; message.set_single_int64(1); @@ -1538,11 +1546,10 @@ TEST_F(DirectSelectStepTest, IdentifiesUnknowns) { RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("test_all_types", -1), - cel::StringValue("single_int64"), - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); + auto step = CreateDirectSelectStep( + CreateDirectIdentStep("test_all_types", -1), "single_int64", + /*test_only=*/false, -1, + /*enable_wrapper_type_null_unboxing=*/true); TestAllTypes message; message.set_single_int64(1); @@ -1573,7 +1580,7 @@ TEST_F(DirectSelectStepTest, ForwardErrorValue) { auto step = CreateDirectSelectStep( CreateConstValueDirectStep(cel::ErrorValue(absl::InternalError("test1")), -1), - cel::StringValue("single_int64"), + "single_int64", /*test_only=*/false, -1, /*enable_wrapper_type_null_unboxing=*/true); @@ -1600,7 +1607,7 @@ TEST_F(DirectSelectStepTest, ForwardUnknownOperand) { CreateConstValueDirectStep(cel::common_internal::MakeUnknownValue( cel::Unknown(std::move(attr_set))), -1), - cel::StringValue("single_int64"), + "single_int64", /*test_only=*/false, -1, /*enable_wrapper_type_null_unboxing=*/true); diff --git a/eval/tests/modern_benchmark_test.cc b/eval/tests/modern_benchmark_test.cc index 8a41b094f..1d7731e6a 100644 --- a/eval/tests/modern_benchmark_test.cc +++ b/eval/tests/modern_benchmark_test.cc @@ -383,9 +383,9 @@ void BM_PolicySymbolic(benchmark::State& state) { ASSERT_OK_AND_ASSIGN(auto cel_expr, runtime->CreateProgram(std::move(ast))); Activation activation; - activation.InsertOrAssignValue("ip", StringValue(&arena, kIP)); - activation.InsertOrAssignValue("path", StringValue(&arena, kPath)); - activation.InsertOrAssignValue("token", StringValue(&arena, kToken)); + activation.InsertOrAssignValue("ip", StringValue::WrapUnsafe(kIP)); + activation.InsertOrAssignValue("path", StringValue::WrapUnsafe(kPath)); + activation.InsertOrAssignValue("token", StringValue::WrapUnsafe(kToken)); for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, @@ -439,11 +439,11 @@ class RequestMapImpl : public CustomMapValueInterface { return false; } if (string_value->Equals("ip")) { - *result = StringValue(kIP); + *result = StringValue::WrapUnsafe(kIP); } else if (string_value->Equals("path")) { - *result = StringValue(kPath); + *result = StringValue::WrapUnsafe(kPath); } else if (string_value->Equals("token")) { - *result = StringValue(kToken); + *result = StringValue::WrapUnsafe(kToken); } else { return false; } @@ -684,9 +684,9 @@ void BM_HasMap(benchmark::State& state) { auto map_builder = cel::NewMapValueBuilder(&arena); - ASSERT_THAT( - map_builder->Put(cel::StringValue("path"), cel::StringValue("path")), - IsOk()); + ASSERT_THAT(map_builder->Put(cel::StringValue::WrapUnsafe("path"), + cel::StringValue::WrapUnsafe("path")), + IsOk()); activation.InsertOrAssignValue("request", std::move(*map_builder).Build()); diff --git a/extensions/encoders.cc b/extensions/encoders.cc index 12ba804e8..4aeb00cca 100644 --- a/extensions/encoders.cc +++ b/extensions/encoders.cc @@ -51,7 +51,7 @@ absl::StatusOr Base64Decode( return ErrorValue::From(absl::InvalidArgumentError("invalid base64 data"), arena); } - return BytesValue(arena, std::move(out)); + return BytesValue::From(std::move(out), arena); } absl::StatusOr Base64Encode( @@ -62,7 +62,7 @@ absl::StatusOr Base64Encode( std::string in; std::string out; out = absl::Base64Escape(value.NativeString(in)); - return StringValue(arena, std::move(out)); + return StringValue::From(std::move(out), arena); } absl::Status RegisterEncodersDecls(TypeCheckerBuilder& builder) { diff --git a/extensions/formatting_test.cc b/extensions/formatting_test.cc index 6a7fb300b..a07dd0bb1 100644 --- a/extensions/formatting_test.cc +++ b/extensions/formatting_test.cc @@ -182,8 +182,8 @@ TEST_P(StringFormatTest, TestStringFormatting) { Activation activation; for (const auto& [name, value] : test_case.dyn_args) { if (std::holds_alternative(value)) { - activation.InsertOrAssignValue(name, - StringValue{std::get(value)}); + activation.InsertOrAssignValue( + name, StringValue::From(std::get(value), &arena)); } else if (std::holds_alternative(value)) { activation.InsertOrAssignValue(name, BoolValue{std::get(value)}); } else if (std::holds_alternative(value)) { diff --git a/extensions/protobuf/value_test.cc b/extensions/protobuf/value_test.cc index 79c15690e..b43c65968 100644 --- a/extensions/protobuf/value_test.cc +++ b/extensions/protobuf/value_test.cc @@ -841,8 +841,9 @@ TEST_F(ProtoValueUnwrapTest, NonMessageValue) { TestAllTypes dest; EXPECT_THAT(ProtoMessageFromValue(IntValue(42), dest), StatusIs(absl::StatusCode::kInvalidArgument)); - EXPECT_THAT(ProtoMessageFromValue(StringValue("not a message"), dest), - StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT( + ProtoMessageFromValue(StringValue::WrapUnsafe("not a message"), dest), + StatusIs(absl::StatusCode::kInvalidArgument)); EXPECT_THAT(ProtoMessageFromValue(NullValue(), dest), StatusIs(absl::StatusCode::kInvalidArgument)); } diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 937ca4737..2524e57d1 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -423,7 +423,7 @@ absl::StatusOr FallbackSelect( } return elem->GetMap().Has( - StringValue(arena, *qualifier.GetStringKey()), + StringValue::WrapUnsafe(*qualifier.GetStringKey()), descriptor_pool, message_factory, arena); }), last_instruction); diff --git a/extensions/select_optimization_test.cc b/extensions/select_optimization_test.cc index d75afe52e..b1213142a 100644 --- a/extensions/select_optimization_test.cc +++ b/extensions/select_optimization_test.cc @@ -1105,8 +1105,8 @@ INSTANTIATE_TEST_SUITE_P( {}, // not set [](google::protobuf::Arena* arena, Activation& act) -> absl::Status { auto builder = cel::NewMapValueBuilder(arena); - CEL_RETURN_IF_ERROR( - builder->Put(cel::StringValue("child"), cel::NullValue())); + CEL_RETURN_IF_ERROR(builder->Put( + cel::StringValue::WrapUnsafe("child"), cel::NullValue())); auto value = std::move(*builder).Build(); diff --git a/runtime/constant_folding_test.cc b/runtime/constant_folding_test.cc index c59d5602a..0770039f3 100644 --- a/runtime/constant_folding_test.cc +++ b/runtime/constant_folding_test.cc @@ -87,9 +87,11 @@ TEST_P(ConstantFoldingExtTest, Runner) { const StringValue&>:: RegisterGlobalOverload( "prepend", - [](const StringValue& value, const StringValue& prefix) { - return StringValue( - absl::StrCat(prefix.ToString(), value.ToString())); + [](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), + context.arena()); }, builder.function_registry()); ASSERT_THAT(status, IsOk()); @@ -150,9 +152,11 @@ TEST(ConstantFoldingExtTest, LazyFunctionNotFolded) { BinaryFunctionAdapter, const StringValue&, const StringValue&>; auto fn = FunctionAdapter::WrapFunction( - [&call_count](const StringValue& value, const StringValue& prefix) { + [&call_count](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { call_count++; - return StringValue(absl::StrCat(prefix.ToString(), value.ToString())); + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), context.arena()); }); FunctionDescriptor descriptor = FunctionAdapter::CreateDescriptor( "lazy_prepend", /*receiver_style=*/false); @@ -189,19 +193,21 @@ TEST(ConstantFoldingExtTest, ContextualFunctionNotFolded) { internal::GetTestingDescriptorPool(), options)); int call_count = 0; - auto status = BinaryFunctionAdapter< - absl::StatusOr, const StringValue&, - const StringValue&>::Register("contextual_prepend", - /*receiver_style=*/false, - [&call_count](const StringValue& value, - const StringValue& prefix) { - call_count++; - return StringValue(absl::StrCat( - prefix.ToString(), value.ToString())); - }, - builder.function_registry(), - {/*.is_strict=*/true, - /*is_contextual=*/true}); + auto status = BinaryFunctionAdapter, const StringValue&, + const StringValue&>:: + Register( + "contextual_prepend", + /*receiver_style=*/false, + [&call_count](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { + call_count++; + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), + context.arena()); + }, + builder.function_registry(), + {/*.is_strict=*/true, + /*is_contextual=*/true}); ASSERT_THAT(status, IsOk()); ASSERT_THAT(EnableConstantFolding(builder), IsOk()); diff --git a/runtime/function_adapter_test.cc b/runtime/function_adapter_test.cc index 910020fdf..df5f50362 100644 --- a/runtime/function_adapter_test.cc +++ b/runtime/function_adapter_test.cc @@ -162,13 +162,14 @@ TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionDuration) { TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionString) { using FunctionAdapter = UnaryFunctionAdapter; - std::unique_ptr wrapped = - FunctionAdapter::WrapFunction([](const StringValue& x) -> StringValue { - return StringValue("pre_" + x.ToString()); + std::unique_ptr wrapped = FunctionAdapter::WrapFunction( + [](const StringValue& x, + const Function::InvokeContext& context) -> StringValue { + return StringValue::From("pre_" + x.ToString(), context.arena()); }); std::vector args; - args.emplace_back() = StringValue("string"); + args.emplace_back() = StringValue::WrapUnsafe("string"); ASSERT_OK_AND_ASSIGN(auto result, wrapped->Invoke(args, test_invoke_context())); @@ -178,13 +179,14 @@ TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionString) { TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionBytes) { using FunctionAdapter = UnaryFunctionAdapter; - std::unique_ptr wrapped = - FunctionAdapter::WrapFunction([](const BytesValue& x) -> BytesValue { - return BytesValue("pre_" + x.ToString()); + std::unique_ptr wrapped = FunctionAdapter::WrapFunction( + [](const BytesValue& x, + const Function::InvokeContext& context) -> BytesValue { + return BytesValue::From("pre_" + x.ToString(), context.arena()); }); std::vector args; - args.emplace_back() = BytesValue("bytes"); + args.emplace_back() = BytesValue::WrapUnsafe("bytes"); ASSERT_OK_AND_ASSIGN(auto result, wrapped->Invoke(args, test_invoke_context())); @@ -480,14 +482,15 @@ TEST_F(FunctionAdapterTest, BinaryFunctionAdapterWrapFunctionString) { BinaryFunctionAdapter, const StringValue&, const StringValue&>; std::unique_ptr wrapped = FunctionAdapter::WrapFunction( - [](const StringValue& x, - const StringValue& y) -> absl::StatusOr { - return StringValue(x.ToString() + y.ToString()); + [](const StringValue& x, const StringValue& y, + const Function::InvokeContext& context) + -> absl::StatusOr { + return StringValue::From(x.ToString() + y.ToString(), context.arena()); }); std::vector args; - args.emplace_back() = StringValue("abc"); - args.emplace_back() = StringValue("def"); + args.emplace_back() = StringValue::WrapUnsafe("abc"); + args.emplace_back() = StringValue::WrapUnsafe("def"); ASSERT_OK_AND_ASSIGN(auto result, wrapped->Invoke(args, test_invoke_context())); @@ -501,14 +504,14 @@ TEST_F(FunctionAdapterTest, BinaryFunctionAdapterWrapFunctionBytes) { BinaryFunctionAdapter, const BytesValue&, const BytesValue&>; std::unique_ptr wrapped = FunctionAdapter::WrapFunction( - [](const BytesValue& x, - const BytesValue& y) -> absl::StatusOr { - return BytesValue(x.ToString() + y.ToString()); + [](const BytesValue& x, const BytesValue& y, + const Function::InvokeContext& context) -> absl::StatusOr { + return BytesValue::From(x.ToString() + y.ToString(), context.arena()); }); std::vector args; - args.emplace_back() = BytesValue("abc"); - args.emplace_back() = BytesValue("def"); + args.emplace_back() = BytesValue::WrapUnsafe("abc"); + args.emplace_back() = BytesValue::WrapUnsafe("def"); ASSERT_OK_AND_ASSIGN(auto result, wrapped->Invoke(args, test_invoke_context())); @@ -712,7 +715,7 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterCreateDescriptor0Args) { TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction0Args) { std::unique_ptr fn = NullaryFunctionAdapter>::WrapFunction( - []() { return StringValue("abc"); }); + []() { return StringValue::WrapUnsafe("abc"); }); ASSERT_OK_AND_ASSIGN(auto result, fn->Invoke({}, descriptor_pool(), message_factory(), arena())); @@ -734,16 +737,18 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterCreateDescriptor3Args) { TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3Args) { std::unique_ptr fn = NaryFunctionAdapter< - absl::StatusOr, int64_t, bool, - const StringValue&>::WrapFunction([](int64_t int_val, bool bool_val, - const StringValue& string_val) - -> absl::StatusOr { - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); - }); + absl::StatusOr, int64_t, bool, const StringValue&>:: + WrapFunction( + [](int64_t int_val, bool bool_val, const StringValue& string_val, + const Function::InvokeContext& context) -> absl::StatusOr { + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); + }); std::vector args{IntValue(42), BoolValue(false)}; - args.emplace_back() = StringValue("abcd"); + args.emplace_back() = StringValue::WrapUnsafe("abcd"); ASSERT_OK_AND_ASSIGN(auto result, fn->Invoke(args, descriptor_pool(), message_factory(), arena())); ASSERT_TRUE(result->Is()); @@ -752,13 +757,15 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3Args) { TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3ArgsBadArgType) { std::unique_ptr fn = NaryFunctionAdapter< - absl::StatusOr, int64_t, bool, - const StringValue&>::WrapFunction([](int64_t int_val, bool bool_val, - const StringValue& string_val) - -> absl::StatusOr { - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); - }); + absl::StatusOr, int64_t, bool, const StringValue&>:: + WrapFunction( + [](int64_t int_val, bool bool_val, const StringValue& string_val, + const Function::InvokeContext& context) -> absl::StatusOr { + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); + }); std::vector args{IntValue(42), BoolValue(false)}; args.emplace_back() = TimestampValue(absl::UnixEpoch()); @@ -769,13 +776,15 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3ArgsBadArgType) { TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction3ArgsBadArgCount) { std::unique_ptr fn = NaryFunctionAdapter< - absl::StatusOr, int64_t, bool, - const StringValue&>::WrapFunction([](int64_t int_val, bool bool_val, - const StringValue& string_val) - -> absl::StatusOr { - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); - }); + absl::StatusOr, int64_t, bool, const StringValue&>:: + WrapFunction( + [](int64_t int_val, bool bool_val, const StringValue& string_val, + const Function::InvokeContext& context) -> absl::StatusOr { + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); + }); std::vector args{IntValue(42), BoolValue(false)}; EXPECT_THAT(fn->Invoke(args, test_invoke_context()), @@ -802,15 +811,17 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction5Args) { absl::StatusOr, int64_t, bool, const StringValue&, int64_t, int64_t>::WrapFunction([](int64_t int_val, bool bool_val, const StringValue& string_val, - int64_t extra_arg, - int64_t extra_arg2) -> absl::StatusOr { - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString(), "_", extra_arg, - "_", extra_arg2)); + int64_t extra_arg, int64_t extra_arg2, + const Function::InvokeContext& context) + -> absl::StatusOr { + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString(), "_", extra_arg, "_", extra_arg2), + context.arena()); }); std::vector args{IntValue(42), BoolValue(false)}; - args.emplace_back() = StringValue("abcd"); + args.emplace_back() = StringValue::WrapUnsafe("abcd"); args.push_back(IntValue(123)); args.push_back(IntValue(456)); ASSERT_OK_AND_ASSIGN(auto result, fn->Invoke(args, descriptor_pool(), @@ -824,12 +835,15 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction5ArgsBadArgType) { absl::StatusOr, int64_t, bool, const StringValue&, int64_t, int64_t>::WrapFunction([](int64_t int_val, bool bool_val, const StringValue& string_val, - int64_t extra_arg, - int64_t extra_arg2) -> absl::StatusOr { + int64_t extra_arg, int64_t extra_arg2, + const Function::InvokeContext& context) + -> absl::StatusOr { static_cast(extra_arg); static_cast(extra_arg2); - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); }); std::vector args{IntValue(42), BoolValue(false)}; @@ -846,12 +860,15 @@ TEST_F(FunctionAdapterTest, NaryFunctionAdapterWrapFunction5ArgsBadArgCount) { absl::StatusOr, int64_t, bool, const StringValue&, int64_t, int64_t>::WrapFunction([](int64_t int_val, bool bool_val, const StringValue& string_val, - int64_t extra_arg, - int64_t extra_arg2) -> absl::StatusOr { + int64_t extra_arg, int64_t extra_arg2, + const Function::InvokeContext& context) + -> absl::StatusOr { static_cast(extra_arg); static_cast(extra_arg2); - return StringValue(absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), - "_", string_val.ToString())); + return StringValue::From( + absl::StrCat(int_val, "_", (bool_val ? "true" : "false"), "_", + string_val.ToString()), + context.arena()); }); std::vector args{IntValue(42), BoolValue(false)}; diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index d8a442f34..f90d24f30 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -87,15 +87,15 @@ cc_library( srcs = ["convert_constant.cc"], hdrs = ["convert_constant.h"], deps = [ - "//common:allocator", - "//common:ast", "//common:constant", "//common:value", "//eval/internal:errors", + "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/time", "@com_google_absl//absl/types:variant", + "@com_google_protobuf//:protobuf", ], ) diff --git a/runtime/internal/convert_constant.cc b/runtime/internal/convert_constant.cc index 33f382858..832b0dfcc 100644 --- a/runtime/internal/convert_constant.cc +++ b/runtime/internal/convert_constant.cc @@ -17,21 +17,22 @@ #include #include +#include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/time/time.h" #include "absl/types/variant.h" -#include "common/allocator.h" #include "common/constant.h" #include "common/value.h" #include "eval/internal/errors.h" +#include "google/protobuf/arena.h" namespace cel::runtime_internal { namespace { using ::cel::Constant; struct ConvertVisitor { - Allocator<> allocator; + google::protobuf::Arena* const arena; absl::StatusOr operator()(std::monostate) { return absl::InvalidArgumentError("unspecified constant"); @@ -48,10 +49,10 @@ struct ConvertVisitor { return DoubleValue(value); } absl::StatusOr operator()(const cel::StringConstant& value) { - return StringValue(allocator, value); + return StringValue::From(value, arena); } absl::StatusOr operator()(const cel::BytesConstant& value) { - return BytesValue(allocator, value); + return BytesValue::From(value, arena); } absl::StatusOr operator()(const absl::Duration duration) { if (duration >= kDurationHigh || duration <= kDurationLow) { @@ -71,8 +72,8 @@ struct ConvertVisitor { // // A status maybe returned if value creation fails. absl::StatusOr ConvertConstant(const Constant& constant, - Allocator<> allocator) { - return absl::visit(ConvertVisitor{allocator}, constant.constant_kind()); + google::protobuf::Arena* absl_nonnull arena) { + return absl::visit(ConvertVisitor{arena}, constant.constant_kind()); } } // namespace cel::runtime_internal diff --git a/runtime/internal/convert_constant.h b/runtime/internal/convert_constant.h index f1ac0c850..e891ca45d 100644 --- a/runtime/internal/convert_constant.h +++ b/runtime/internal/convert_constant.h @@ -14,10 +14,11 @@ #ifndef THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_CONVERT_CONSTANT_H_ #define THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_CONVERT_CONSTANT_H_ +#include "absl/base/nullability.h" #include "absl/status/statusor.h" -#include "common/allocator.h" -#include "common/ast.h" +#include "common/constant.h" #include "common/value.h" +#include "google/protobuf/arena.h" namespace cel::runtime_internal { @@ -32,7 +33,7 @@ namespace cel::runtime_internal { // A status may still be returned if value creation fails according to // value_factory's policy. absl::StatusOr ConvertConstant(const Constant& constant, - Allocator<> allocator); + google::protobuf::Arena* absl_nonnull arena); } // namespace cel::runtime_internal diff --git a/runtime/internal/function_adapter_test.cc b/runtime/internal/function_adapter_test.cc index 643f08090..d57620999 100644 --- a/runtime/internal/function_adapter_test.cc +++ b/runtime/internal/function_adapter_test.cc @@ -179,7 +179,7 @@ TEST_F(ValueToAdaptedVisitorTest, DurationWrongKind) { } TEST_F(ValueToAdaptedVisitorTest, String) { - Value v = cel::StringValue("string"); + Value v = cel::StringValue::WrapUnsafe("string"); StringValue out; ASSERT_THAT(ValueToAdaptedVisitor{v}(&out), IsOk()); @@ -197,7 +197,7 @@ TEST_F(ValueToAdaptedVisitorTest, StringWrongKind) { } TEST_F(ValueToAdaptedVisitorTest, Bytes) { - Value v = cel::BytesValue("bytes"); + Value v = cel::BytesValue::WrapUnsafe("bytes"); BytesValue out; ASSERT_THAT(ValueToAdaptedVisitor{v}(&out), IsOk()); @@ -272,7 +272,7 @@ TEST_F(AdaptedToValueVisitorTest, Duration) { } TEST_F(AdaptedToValueVisitorTest, String) { - StringValue value = cel::StringValue("str"); + StringValue value = cel::StringValue::WrapUnsafe("str"); ASSERT_OK_AND_ASSIGN(auto result, AdaptedToValueVisitor{}(value)); @@ -281,7 +281,7 @@ TEST_F(AdaptedToValueVisitorTest, String) { } TEST_F(AdaptedToValueVisitorTest, Bytes) { - BytesValue value = cel::BytesValue("bytes"); + BytesValue value = cel::BytesValue::WrapUnsafe("bytes"); ASSERT_OK_AND_ASSIGN(auto result, AdaptedToValueVisitor{}(value)); diff --git a/runtime/regex_precompilation_test.cc b/runtime/regex_precompilation_test.cc index 85b47ef45..f31d34455 100644 --- a/runtime/regex_precompilation_test.cc +++ b/runtime/regex_precompilation_test.cc @@ -87,9 +87,11 @@ TEST_P(RegexPrecompilationTest, Basic) { absl::StatusOr, const StringValue&, const StringValue&>>:: RegisterGlobalOverload( "prepend", - [](const StringValue& value, const StringValue& prefix) { - return StringValue( - absl::StrCat(prefix.ToString(), value.ToString())); + [](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), + context.arena()); }, builder.function_registry()); ASSERT_THAT(status, IsOk()); @@ -114,7 +116,7 @@ TEST_P(RegexPrecompilationTest, Basic) { google::protobuf::Arena arena; Activation activation; activation.InsertOrAssignValue("string_var", - StringValue(&arena, "string_var")); + StringValue::WrapUnsafe("string_var")); ASSERT_OK_AND_ASSIGN(Value value, program->Evaluate(&arena, activation)); EXPECT_THAT(value, test_case.result_matcher); @@ -131,9 +133,11 @@ TEST_P(RegexPrecompilationTest, WithConstantFolding) { absl::StatusOr, const StringValue&, const StringValue&>>:: RegisterGlobalOverload( "prepend", - [](const StringValue& value, const StringValue& prefix) { - return StringValue( - absl::StrCat(prefix.ToString(), value.ToString())); + [](const StringValue& value, const StringValue& prefix, + const Function::InvokeContext& context) { + return StringValue::From( + absl::StrCat(prefix.ToString(), value.ToString()), + context.arena()); }, builder.function_registry()); ASSERT_THAT(status, IsOk()); @@ -158,7 +162,7 @@ TEST_P(RegexPrecompilationTest, WithConstantFolding) { google::protobuf::Arena arena; Activation activation; activation.InsertOrAssignValue("string_var", - StringValue(&arena, "string_var")); + StringValue::WrapUnsafe("string_var")); ASSERT_OK_AND_ASSIGN(Value value, program->Evaluate(&arena, activation)); EXPECT_THAT(value, test_case.result_matcher); diff --git a/runtime/standard/type_conversion_functions.cc b/runtime/standard/type_conversion_functions.cc index f99f07f31..2c5130aa3 100644 --- a/runtime/standard/type_conversion_functions.cc +++ b/runtime/standard/type_conversion_functions.cc @@ -193,7 +193,7 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, absl::InvalidArgumentError("malformed UTF-8 bytes"), context.arena()); } - return StringValue(value.ToString()); + return StringValue(value); }, registry); CEL_RETURN_IF_ERROR(status); @@ -202,7 +202,7 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, [](bool value) -> StringValue { - return StringValue(value ? "true" : "false"); + return StringValue::WrapUnsafe(value ? "true" : "false"); }, registry); CEL_RETURN_IF_ERROR(status); @@ -218,8 +218,8 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, // int -> string status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](int64_t value) -> StringValue { - return StringValue(absl::StrCat(value)); + [](int64_t value, const Function::InvokeContext& context) -> StringValue { + return StringValue::From(absl::StrCat(value), context.arena()); }, registry); CEL_RETURN_IF_ERROR(status); @@ -234,8 +234,9 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, // uint -> string status = UnaryFunctionAdapter::RegisterGlobalOverload( cel::builtin::kString, - [](uint64_t value) -> StringValue { - return StringValue(absl::StrCat(value)); + [](uint64_t value, + const Function::InvokeContext& context) -> StringValue { + return StringValue::From(absl::StrCat(value), context.arena()); }, registry); CEL_RETURN_IF_ERROR(status); @@ -249,7 +250,7 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, if (!encode.ok()) { return ErrorValue::From(encode.status(), context.arena()); } - return StringValue(*encode); + return StringValue::From(*encode, context.arena()); }, registry); CEL_RETURN_IF_ERROR(status); @@ -262,7 +263,7 @@ absl::Status RegisterStringConversionFunctions(FunctionRegistry& registry, if (!encode.ok()) { return ErrorValue::From(encode.status(), context.arena()); } - return StringValue(*encode); + return StringValue::From(*encode, context.arena()); }, registry); } @@ -332,8 +333,7 @@ absl::Status RegisterBytesConversionFunctions(FunctionRegistry& registry, return UnaryFunctionAdapter, const StringValue&>:: RegisterGlobalOverload( cel::builtin::kBytes, - [](const StringValue& value) { return BytesValue(value.ToString()); }, - registry); + [](const StringValue& value) { return BytesValue(value); }, registry); } absl::Status RegisterDoubleConversionFunctions(FunctionRegistry& registry, From 43674f3afcaa893e4cef6fd01f106a9f05c0d488 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 28 Sep 2026 15:56:20 -0700 Subject: [PATCH 242/350] Refactor expression step representation. Moves the id and a small metadata header inline in the main program vector. No functional changes. PiperOrigin-RevId: 989892858 --- .../cel_expression_builder_flat_impl.cc | 7 +- eval/compiler/constant_folding.cc | 6 +- eval/compiler/constant_folding_test.cc | 128 +++++++-------- eval/compiler/flat_expr_builder.cc | 134 +++++++++------- eval/compiler/flat_expr_builder_extensions.cc | 37 +++-- eval/compiler/flat_expr_builder_extensions.h | 27 ++-- .../flat_expr_builder_extensions_test.cc | 62 ++++---- eval/compiler/instrumentation.cc | 3 +- .../regex_precompilation_optimization.cc | 14 +- eval/eval/cel_expression_flat_impl.cc | 9 +- eval/eval/cel_expression_flat_impl.h | 2 +- eval/eval/compiler_constant_step.h | 3 +- eval/eval/compiler_constant_step_test.cc | 10 +- eval/eval/comprehension_step.cc | 10 +- eval/eval/comprehension_step.h | 16 +- eval/eval/comprehension_step_test.cc | 46 +++--- eval/eval/const_value_step.h | 7 +- eval/eval/container_access_step.cc | 11 +- eval/eval/container_access_step.h | 5 +- eval/eval/container_access_step_test.cc | 27 ++-- eval/eval/create_list_step.cc | 25 ++- eval/eval/create_list_step.h | 6 +- eval/eval/create_list_step_test.cc | 48 +++--- eval/eval/create_map_step.cc | 22 ++- eval/eval/create_map_step.h | 9 +- eval/eval/create_map_step_test.cc | 22 ++- eval/eval/create_struct_step.cc | 12 +- eval/eval/create_struct_step.h | 4 +- eval/eval/create_struct_step_test.cc | 26 ++- eval/eval/direct_expression_step.h | 7 +- eval/eval/equality_steps.cc | 14 +- eval/eval/equality_steps.h | 5 +- eval/eval/equality_steps_test.cc | 51 +++--- eval/eval/evaluator_core.cc | 13 +- eval/eval/evaluator_core.h | 150 +++++++++++++++--- eval/eval/evaluator_core_test.cc | 89 ++++++++--- eval/eval/expression_step_base.h | 9 +- eval/eval/function_step.cc | 10 +- eval/eval/function_step.h | 4 +- eval/eval/function_step_test.cc | 63 ++++---- eval/eval/ident_step.cc | 19 ++- eval/eval/ident_step.h | 7 +- eval/eval/ident_step_test.cc | 45 +++--- eval/eval/jump_step.cc | 33 ++-- eval/eval/jump_step.h | 14 +- eval/eval/lazy_init_step.cc | 35 ++-- eval/eval/lazy_init_step.h | 16 +- eval/eval/lazy_init_step_test.cc | 22 +-- eval/eval/logic_step.cc | 30 ++-- eval/eval/logic_step.h | 10 +- eval/eval/logic_step_test.cc | 15 +- eval/eval/optional_or_step.cc | 17 +- eval/eval/optional_or_step.h | 6 +- eval/eval/regex_match_step.cc | 11 +- eval/eval/regex_match_step.h | 5 +- eval/eval/select_step.cc | 38 ++--- eval/eval/select_step.h | 10 +- eval/eval/select_step_test.cc | 85 +++++----- eval/eval/shadowable_value_step.cc | 10 +- eval/eval/shadowable_value_step.h | 4 +- eval/eval/shadowable_value_step_test.cc | 21 ++- eval/eval/ternary_step.cc | 7 +- eval/eval/ternary_step.h | 3 +- eval/eval/ternary_step_test.cc | 79 ++++----- extensions/select_optimization.cc | 6 +- runtime/internal/runtime_impl.cc | 10 +- 66 files changed, 925 insertions(+), 786 deletions(-) diff --git a/eval/compiler/cel_expression_builder_flat_impl.cc b/eval/compiler/cel_expression_builder_flat_impl.cc index 98ecc6aae..830b5ff1d 100644 --- a/eval/compiler/cel_expression_builder_flat_impl.cc +++ b/eval/compiler/cel_expression_builder_flat_impl.cc @@ -100,7 +100,12 @@ CelExpressionBuilderFlatImpl::CreateExpressionImpl( !impl.subexpressions().empty() && // mainline expression is exactly one recursive step. impl.subexpressions().front().size() == 1 && - impl.subexpressions().front().front()->GetNativeTypeId() == + impl.subexpressions().front().front().IsGenericStep() && + impl.subexpressions() + .front() + .front() + .GetGenericStep() + ->GetNativeTypeId() == cel::NativeTypeId::For()) { return CelExpressionRecursiveImpl::Create(env_, std::move(impl)); } diff --git a/eval/compiler/constant_folding.cc b/eval/compiler/constant_folding.cc index fa12a8e6d..93cb35e65 100644 --- a/eval/compiler/constant_folding.cc +++ b/eval/compiler/constant_folding.cc @@ -55,6 +55,7 @@ using ::google::api::expr::runtime::EvaluationListener; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionPath; using ::google::api::expr::runtime::ExecutionPathView; +using ::google::api::expr::runtime::ExpressionStep; using ::google::api::expr::runtime::FlatExpressionEvaluatorState; using ::google::api::expr::runtime::PlannerContext; using ::google::api::expr::runtime::ProgramOptimizer; @@ -242,9 +243,8 @@ absl::Status ConstantFoldingExtension::OnPostVisit(PlannerContext& context, // Otherwise make a stack machine plan. ExecutionPath new_plan; - CEL_ASSIGN_OR_RETURN( - new_plan.emplace_back(), - CreateConstValueStep(std::move(value), node.id(), false)); + new_plan.push_back(ExpressionStep::MakeGenericStep( + CreateConstValueStep(std::move(value)), node.id())); return context.ReplaceSubplan(node, std::move(new_plan)); } diff --git a/eval/compiler/constant_folding_test.cc b/eval/compiler/constant_folding_test.cc index d1c0c31e0..1528f5a5d 100644 --- a/eval/compiler/constant_folding_test.cc +++ b/eval/compiler/constant_folding_test.cc @@ -62,6 +62,7 @@ using ::google::api::expr::runtime::CreateConstValueStep; using ::google::api::expr::runtime::CreateCreateListStep; using ::google::api::expr::runtime::CreateCreateStructStepForMap; using ::google::api::expr::runtime::ExecutionPath; +using ::google::api::expr::runtime::ExpressionStep; using ::google::api::expr::runtime::PlannerContext; using ::google::api::expr::runtime::ProgramBuilder; using ::google::api::expr::runtime::ProgramOptimizer; @@ -115,26 +116,25 @@ TEST_F(UpdatedConstantFoldingTest, SkipsTernary) { program_builder.EnterSubexpression(&call); // condition program_builder.EnterSubexpression(&condition); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(true)), condition.id())); program_builder.ExitSubexpression(&condition); // true program_builder.EnterSubexpression(&true_branch); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(true)), true_branch.id())); program_builder.ExitSubexpression(&true_branch); // false program_builder.EnterSubexpression(&false_branch); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(true)), false_branch.id())); program_builder.ExitSubexpression(&false_branch); // ternary. - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::NullValue()), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -179,21 +179,20 @@ TEST_F(UpdatedConstantFoldingTest, SkipsOr) { // left program_builder.EnterSubexpression(&left_condition); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::BoolValue(false), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(false)), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(true)), right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::NullValue()), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -235,21 +234,20 @@ TEST_F(UpdatedConstantFoldingTest, SkipsAnd) { // left program_builder.EnterSubexpression(&left_condition); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(true)), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(false), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(false)), right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::NullValue()), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -291,19 +289,21 @@ TEST_F(UpdatedConstantFoldingTest, CreatesList) { // elem one program_builder.EnterSubexpression(&elem_one); - ASSERT_OK_AND_ASSIGN(auto step, CreateConstValueStep(cel::IntValue(1L), 1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(1L)), elem_one.id())); program_builder.ExitSubexpression(&elem_one); // elem two program_builder.EnterSubexpression(&elem_two); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(2L), 2)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(2L)), elem_two.id())); program_builder.ExitSubexpression(&elem_two); // createlist - ASSERT_OK_AND_ASSIGN(step, CreateCreateListStep(create_list.list_expr(), 3)); - program_builder.AddStep(std::move(step)); + ASSERT_OK_AND_ASSIGN(auto step, + CreateCreateListStep(create_list.list_expr())); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(step), create_list.id())); program_builder.ExitSubexpression(&create_list); std::shared_ptr arena; @@ -349,37 +349,39 @@ TEST_F(UpdatedConstantFoldingTest, CreatesLargeList) { // 0 ASSERT_TRUE(program_builder.EnterSubexpression(&elem0) != nullptr); - ASSERT_OK_AND_ASSIGN(auto step, CreateConstValueStep(cel::IntValue(1L), 1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(1L)), elem0.id())); program_builder.ExitSubexpression(&elem0); // 1 ASSERT_TRUE(program_builder.EnterSubexpression(&elem1)); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(2L), 2)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(2L)), elem1.id())); program_builder.ExitSubexpression(&elem1); // 2 ASSERT_TRUE(program_builder.EnterSubexpression(&elem2) != nullptr); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(3L), 3)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(3L)), elem2.id())); program_builder.ExitSubexpression(&elem2); // 3 ASSERT_TRUE(program_builder.EnterSubexpression(&elem3) != nullptr); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(4L), 4)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(4L)), elem3.id())); program_builder.ExitSubexpression(&elem3); // 4 ASSERT_TRUE(program_builder.EnterSubexpression(&elem4) != nullptr); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(5L), 5)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(5L)), elem4.id())); program_builder.ExitSubexpression(&elem4); // createlist - ASSERT_OK_AND_ASSIGN(step, CreateCreateListStep(create_list.list_expr(), 6)); - program_builder.AddStep(std::move(step)); + ASSERT_OK_AND_ASSIGN(auto step_large, + CreateCreateListStep(create_list.list_expr())); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(step_large), create_list.id())); program_builder.ExitSubexpression(&create_list); std::shared_ptr arena; @@ -426,21 +428,22 @@ TEST_F(UpdatedConstantFoldingTest, CreatesMap) { // key program_builder.EnterSubexpression(&key); - ASSERT_OK_AND_ASSIGN(auto step, CreateConstValueStep(cel::IntValue(1L), 1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(1L)), key.id())); program_builder.ExitSubexpression(&key); // value program_builder.EnterSubexpression(&value); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(2L), 2)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(2L)), value.id())); program_builder.ExitSubexpression(&value); // create map ASSERT_OK_AND_ASSIGN( - step, CreateCreateStructStepForMap(create_map.map_expr().entries().size(), - {}, 3)); - program_builder.AddStep(std::move(step)); + auto step_map, + CreateCreateStructStepForMap(create_map.map_expr().entries().size(), {})); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(step_map), create_map.id())); program_builder.ExitSubexpression(&create_map); std::shared_ptr arena; @@ -481,22 +484,22 @@ TEST_F(UpdatedConstantFoldingTest, CreatesInvalidMap) { // key program_builder.EnterSubexpression(&key); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::DoubleValue(1.0), 1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::DoubleValue(1.0)), key.id())); program_builder.ExitSubexpression(&key); // value program_builder.EnterSubexpression(&value); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::IntValue(2L), 2)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::IntValue(2L)), value.id())); program_builder.ExitSubexpression(&value); // create map ASSERT_OK_AND_ASSIGN( - step, CreateCreateStructStepForMap(create_map.map_expr().entries().size(), - {}, 3)); - program_builder.AddStep(std::move(step)); + auto step_invalid_map, + CreateCreateStructStepForMap(create_map.map_expr().entries().size(), {})); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + std::move(step_invalid_map), create_map.id())); program_builder.ExitSubexpression(&create_map); std::shared_ptr arena; @@ -536,21 +539,20 @@ TEST_F(UpdatedConstantFoldingTest, ErrorsOnUnexpectedOrder) { program_builder.EnterSubexpression(&call); // left program_builder.EnterSubexpression(&left_condition); - ASSERT_OK_AND_ASSIGN(auto step, - CreateConstValueStep(cel::BoolValue(true), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(true)), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::BoolValue(false), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::BoolValue(false)), right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - ASSERT_OK_AND_ASSIGN(step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(step)); + program_builder.AddStep(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::NullValue()), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 5a2490976..677918c7c 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -730,8 +730,8 @@ class FlatExprVisitor : public cel::AstVisitor { return; } - AddStep( - CreateConstValueStep(std::move(converted_value).value(), expr.id())); + AddStep(CreateConstValueStep(std::move(converted_value).value()), + expr.id()); } struct SlotLookupResult { @@ -855,8 +855,8 @@ class FlatExprVisitor : public cel::AstVisitor { program.depth + 1); } else { // Off by one since mainline expression will be index 0. - AddStep( - CreateLazyInitStep(slot.slot, slot.subexpression + 1, expr.id())); + AddStep(CreateLazyInitStep(slot.slot, slot.subexpression + 1), + expr.id()); } return; } else if (slot.slot >= 0) { @@ -865,8 +865,8 @@ class FlatExprVisitor : public cel::AstVisitor { CreateDirectSlotIdentStep(ident_expr.name(), slot.slot, expr.id()), 1); } else { - AddStep( - CreateIdentStepForSlot(ident_expr.name(), slot.slot, expr.id())); + AddStep(CreateIdentStepForSlot(ident_expr.name(), slot.slot), + expr.id()); } return; } @@ -918,8 +918,8 @@ class FlatExprVisitor : public cel::AstVisitor { 1); return; } - AddStep(CreateShadowableValueStep(name, std::move(const_value).value(), - select_root_id)); + AddStep(CreateShadowableValueStep(name, std::move(const_value).value()), + select_root_id); return; } @@ -927,7 +927,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (options_.max_recursion_depth != 0) { SetRecursiveStep(CreateDirectIdentStep(ident_name, expr.id()), 1); } else { - AddStep(CreateIdentStep(ident_name, expr.id())); + AddStep(CreateIdentStep(ident_name), expr.id()); } } @@ -1032,15 +1032,18 @@ class FlatExprVisitor : public cel::AstVisitor { } if (field_type.has_value()) { - AddStep(CreateTypedSelectStep( - std::move(field), *struct_type, *std::move(field_type), - select_expr.test_only(), expr.id(), - options_.enable_empty_wrapper_null_unboxing, enable_optional_types_)); + AddStep( + CreateTypedSelectStep(std::move(field), *struct_type, + *std::move(field_type), select_expr.test_only(), + options_.enable_empty_wrapper_null_unboxing, + enable_optional_types_), + expr.id()); return; } - AddStep(CreateSelectStep( - std::move(field), select_expr.test_only(), expr.id(), - options_.enable_empty_wrapper_null_unboxing, enable_optional_types_)); + AddStep(CreateSelectStep(std::move(field), select_expr.test_only(), + options_.enable_empty_wrapper_null_unboxing, + enable_optional_types_), + expr.id()); } // Call node handler group. @@ -1577,7 +1580,7 @@ class FlatExprVisitor : public cel::AstVisitor { SetRecursiveStep(CreateDirectMutableListStep(expr.id()), 1); return; } - AddStep(CreateMutableListStep(expr.id())); + AddStep(CreateMutableListStep(), expr.id()); return; } if (GetOptimizableListAppendOperand(comprehension.comprehension) == @@ -1598,7 +1601,7 @@ class FlatExprVisitor : public cel::AstVisitor { SetRecursiveStep(std::move(step), *depth + 1); return; } - AddStep(CreateCreateListStep(list_expr, expr.id())); + AddStep(CreateCreateListStep(list_expr), expr.id()); } // CreateStruct node handler. @@ -1635,8 +1638,8 @@ class FlatExprVisitor : public cel::AstVisitor { } AddStep(CreateCreateStructStep(std::move(resolved_name), std::move(fields), - MakeOptionalIndicesSet(struct_expr), - expr.id())); + MakeOptionalIndicesSet(struct_expr)), + expr.id()); } void PostVisitMap(const cel::Expr& expr, @@ -1655,7 +1658,7 @@ class FlatExprVisitor : public cel::AstVisitor { SetRecursiveStep(CreateDirectMutableMapStep(expr.id()), 1); return; } - AddStep(CreateMutableMapStep(expr.id())); + AddStep(CreateMutableMapStep(), expr.id()); return; } } @@ -1674,8 +1677,8 @@ class FlatExprVisitor : public cel::AstVisitor { return; } AddStep(CreateCreateStructStepForMap(map_expr.entries().size(), - MakeOptionalIndicesSet(map_expr), - expr.id())); + MakeOptionalIndicesSet(map_expr)), + expr.id()); } absl::Status progress_status() const { return progress_status_; } @@ -1708,8 +1711,9 @@ class FlatExprVisitor : public cel::AstVisitor { *depth + 1); return; } - AddStep(CreateFunctionStep(*call_expr, expr->id(), - std::move(lazy_overloads))); + AddStep( + CreateFunctionStep(*call_expr, expr->id(), std::move(lazy_overloads)), + expr->id()); return; } @@ -1743,7 +1747,8 @@ class FlatExprVisitor : public cel::AstVisitor { *recursion_depth + 1); return; } - AddStep(CreateFunctionStep(*call_expr, expr->id(), std::move(overloads))); + AddStep(CreateFunctionStep(*call_expr, expr->id(), std::move(overloads)), + expr->id()); } // Add a step to the program, taking ownership. If successful, returns the @@ -1752,21 +1757,32 @@ class FlatExprVisitor : public cel::AstVisitor { // Note: the pointer is only guaranteed to stay valid until the parent // subexpression is finalized. Optimizers may modify the program plan which // may free the step at that point. - ExpressionStep* AddStep( - absl::StatusOr> step) { + template + std::enable_if_t, T*> AddStep( + std::unique_ptr step, int64_t expr_id = -1) { + if (progress_status_.ok() && !PlanningSuppressed()) { + T* ptr = step.get(); + program_builder_.AddStep( + ExpressionStep::MakeGenericStep(std::move(step), expr_id)); + return ptr; + } + return nullptr; + } + + template + std::enable_if_t, T*> AddStep( + absl::StatusOr> step, int64_t expr_id = -1) { if (step.ok()) { - return AddStep(*std::move(step)); + return AddStep(*std::move(step), expr_id); } else { SetProgressStatusIfError(step.status()); } return nullptr; } - template - std::enable_if_t, T*> AddStep( - std::unique_ptr step) { + ExpressionStep* AddStep(ExpressionStep step) { if (progress_status_.ok() && !PlanningSuppressed()) { - return static_cast(program_builder_.AddStep(std::move(step))); + return program_builder_.AddStep(std::move(step)); } return nullptr; } @@ -2026,8 +2042,8 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleIndex( *depth + 1); return CallHandlerResult::kIntercepted; } - AddStep( - CreateContainerAccessStep(call_expr, expr.id(), enable_optional_types_)); + AddStep(CreateContainerAccessStep(call_expr, enable_optional_types_), + expr.id()); return CallHandlerResult::kIntercepted; } @@ -2051,7 +2067,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNot( *depth + 1); return CallHandlerResult::kIntercepted; } - AddStep(CreateNotStep(expr.id())); + AddStep(CreateNotStep(), expr.id()); return CallHandlerResult::kIntercepted; } @@ -2076,7 +2092,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNotStrictlyFalse( *depth + 1); return CallHandlerResult::kIntercepted; } - AddStep(CreateNotStrictlyFalseStep(expr.id())); + AddStep(CreateNotStrictlyFalseStep(), expr.id()); return CallHandlerResult::kIntercepted; } @@ -2118,7 +2134,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleBlock( // Otherwise, iterative plan. if (block.slot_count > 0) { - AddStep(CreateClearSlotsStep(block.index, block.slot_count, expr.id())); + AddStep(CreateClearSlotsStep(block.index, block.slot_count), expr.id()); } return CallHandlerResult::kIntercepted; @@ -2180,7 +2196,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleHeterogeneousEquality( *depth + 1); return CallHandlerResult::kIntercepted; } - AddStep(CreateEqualityStep(inequality, expr.id())); + AddStep(CreateEqualityStep(inequality), expr.id()); return CallHandlerResult::kIntercepted; } @@ -2205,7 +2221,7 @@ FlatExprVisitor::HandleHeterogeneousEqualityIn(const cel::Expr& expr, return CallHandlerResult::kIntercepted; } - AddStep(CreateInStep(expr.id())); + AddStep(CreateInStep(), expr.id()); return CallHandlerResult::kIntercepted; } @@ -2240,9 +2256,9 @@ void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { const size_t num_args = expr->call_expr().args().size(); if (arg_num == last_arg_index) { if (is_or_) { - visitor_->AddStep(CreateOrStep(num_args, expr->id())); + visitor_->AddStep(CreateOrStep(num_args), expr->id()); } else { - visitor_->AddStep(CreateAndStep(num_args, expr->id())); + visitor_->AddStep(CreateAndStep(num_args), expr->id()); } if (short_circuiting_ && !jump_steps_.empty()) { for (auto& jump : jump_steps_) { @@ -2254,10 +2270,9 @@ void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { if (short_circuiting_ && arg_num < last_arg_index) { std::unique_ptr jump_step = is_or_ - ? CreateCondJumpStep(true, {}, /*expected_stack_size=*/arg_num + 1, - expr->id()) - : CreateCondJumpStep(false, {}, /*expected_stack_size=*/arg_num + 1, - expr->id()); + ? CreateCondJumpStep(true, {}, /*expected_stack_size=*/arg_num + 1) + : CreateCondJumpStep(false, {}, + /*expected_stack_size=*/arg_num + 1); ProgramStepIndex index = visitor_->GetCurrentIndex(); if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); jump_step_ptr) { @@ -2289,7 +2304,7 @@ void OptionalOrCondVisitor::PostVisitTarget(const cel::Expr* expr) { // Retain a pointer to the jump step so we can update the target after // planning the second argument. std::unique_ptr jump_step = - CreateOptionalHasValueJumpStep(is_or_value_, expr->id()); + CreateOptionalHasValueJumpStep(is_or_value_); ProgramStepIndex index = visitor_->GetCurrentIndex(); if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); jump_step_ptr) { @@ -2304,7 +2319,7 @@ void OptionalOrCondVisitor::PostVisit(const cel::Expr* expr) { return; } - visitor_->AddStep(CreateOptionalOrStep(is_or_value_, expr->id())); + visitor_->AddStep(CreateOptionalOrStep(is_or_value_), expr->id()); if (short_circuiting_) { for (auto& jump : jump_steps_) { visitor_->SetProgressStatusIfError( @@ -2338,8 +2353,7 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { if (arg_num == 0) { // Jump in case of error or non-bool ProgramStepIndex error_jump_pos = visitor_->GetCurrentIndex(); - auto* error_jump = - visitor_->AddStep(CreateBoolCheckJumpStep({}, expr->id())); + auto* error_jump = visitor_->AddStep(CreateBoolCheckJumpStep()); if (error_jump) { error_jump_ = Jump(error_jump_pos, error_jump); } @@ -2347,8 +2361,7 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { // Jump to the second branch of execution // Value is to be removed from the stack. ProgramStepIndex cond_jump_pos = visitor_->GetCurrentIndex(); - auto* jump_to_second = - visitor_->AddStep(CreateTernaryCondJumpStep({}, expr->id())); + auto* jump_to_second = visitor_->AddStep(CreateTernaryCondJumpStep()); if (jump_to_second) { jump_to_second_ = Jump(cond_jump_pos, static_cast(jump_to_second)); @@ -2357,7 +2370,7 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { // Jump after the first and over the second branch of execution. // Value is to be removed from the stack. ProgramStepIndex jump_pos = visitor_->GetCurrentIndex(); - auto* jump_after_first = visitor_->AddStep(CreateJumpStep({}, expr->id())); + auto* jump_after_first = visitor_->AddStep(CreateJumpStep()); if (!jump_after_first) { return; } @@ -2406,7 +2419,7 @@ void ExhaustiveTernaryCondVisitor::PostVisit(const cel::Expr* expr) { visitor_->MakeTernaryRecursive(expr); return; } - visitor_->AddStep(CreateTernaryStep(expr->id())); + visitor_->AddStep(CreateTernaryStep(), expr->id()); } void ComprehensionVisitor::PreVisit(const cel::Expr* expr) { @@ -2425,25 +2438,24 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( switch (arg_num) { case cel::ITER_RANGE: { init_step_pos_ = visitor_->GetCurrentIndex(); - init_step_ = visitor_->AddStep( - std::make_unique(expr->id())); + init_step_ = visitor_->AddStep(std::make_unique()); break; } case cel::ACCU_INIT: { next_step_pos_ = visitor_->GetCurrentIndex(); next_step_ = visitor_->AddStep(std::make_unique( - iter_slot_, iter2_slot_, accu_slot_, expr->id())); + iter_slot_, iter2_slot_, accu_slot_)); break; } case cel::LOOP_CONDITION: { cond_step_pos_ = visitor_->GetCurrentIndex(); cond_step_ = visitor_->AddStep(std::make_unique( - iter_slot_, iter2_slot_, accu_slot_, short_circuiting_, expr->id())); + iter_slot_, iter2_slot_, accu_slot_, short_circuiting_)); break; } case cel::LOOP_STEP: { ProgramStepIndex index = visitor_->GetCurrentIndex(); - auto* jump_to_next = visitor_->AddStep(CreateJumpStep({}, expr->id())); + auto* jump_to_next = visitor_->AddStep(CreateJumpStep()); if (!jump_to_next) { break; } @@ -2473,7 +2485,7 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( // Encountered an error earlier. Can't determine where to jump. break; } - visitor_->AddStep(CreateComprehensionFinishStep(accu_slot_, expr->id())); + visitor_->AddStep(CreateComprehensionFinishStep(accu_slot_), expr->id()); // Set offsets jumping past the result step in case of errors. CEL_ASSIGN_OR_RETURN( int jump_from_init, @@ -2517,7 +2529,7 @@ void ComprehensionVisitor::PostVisitArgTrivial(cel::ComprehensionArg arg_num, break; } case cel::RESULT: { - visitor_->AddStep(CreateClearSlotStep(accu_slot_, expr->id())); + visitor_->AddStep(CreateClearSlotStep(accu_slot_), expr->id()); break; } } diff --git a/eval/compiler/flat_expr_builder_extensions.cc b/eval/compiler/flat_expr_builder_extensions.cc index ee106ff4a..7e1417eb7 100644 --- a/eval/compiler/flat_expr_builder_extensions.cc +++ b/eval/compiler/flat_expr_builder_extensions.cc @@ -205,7 +205,7 @@ void Subexpression::Flatten() { return; } - std::vector> flat; + ExecutionPath flat; std::vector flatten_stack; @@ -221,8 +221,10 @@ void Subexpression::Flatten() { elements.clear(); continue; } else if (subexpr->IsRecursive()) { - flat.push_back(std::make_unique( - std::move(subexpr->ExtractRecursiveProgram().step), + flat.push_back(ExpressionStep::MakeGenericStep( + std::make_unique( + std::move(subexpr->ExtractRecursiveProgram().step), + subexpr->self_->id()), subexpr->self_->id())); continue; } @@ -237,8 +239,7 @@ void Subexpression::Flatten() { flatten_stack.push_back({subexpr, i + 1}); flatten_stack.push_back({*child, 0}); break; - } else if (auto* step = - absl::get_if>(&element); + } else if (auto* step = absl::get_if(&element); step != nullptr) { flat.push_back(std::move(*step)); } else { @@ -259,8 +260,7 @@ Subexpression::RecursiveProgram Subexpression::ExtractRecursiveProgram() { return result; } -bool Subexpression::ExtractTo( - std::vector>& out) { +bool Subexpression::ExtractTo(ExecutionPath& out) { if (!IsFlattened()) { return false; } @@ -272,9 +272,8 @@ bool Subexpression::ExtractTo( return true; } -std::vector> -ProgramBuilder::FlattenSubexpression(Subexpression* expr) { - std::vector> out; +ExecutionPath ProgramBuilder::FlattenSubexpression(Subexpression* expr) { + ExecutionPath out; if (!expr) { return out; @@ -347,13 +346,19 @@ Subexpression* absl_nullable ProgramBuilder::GetSubexpression( return it->second.get(); } -ExpressionStep* absl_nullable ProgramBuilder::AddStep( - std::unique_ptr step) { +ExpressionStep* absl_nullable ProgramBuilder::AddStep(ExpressionStep step) { if (current_ == nullptr) { return nullptr; } - auto* step_ptr = step.get(); - return current_->AddStep(std::move(step)) ? step_ptr : nullptr; + if (current_->IsRecursive()) { + return nullptr; + } + if (current_->IsFlattened()) { + current_->flattened_elements().push_back(std::move(step)); + return ¤t_->flattened_elements().back(); + } + auto& elem = current_->elements().emplace_back(std::move(step)); + return absl::get_if(&elem); } int ProgramBuilder::ExtractSubexpression(const cel::Expr* expr) { @@ -457,8 +462,8 @@ absl::Status PlannerContext::ReplaceSubplan( return absl::OkStatus(); } -absl::Status PlannerContext::AddSubplanStep( - const cel::Expr& node, std::unique_ptr step) { +absl::Status PlannerContext::AddSubplanStep(const cel::Expr& node, + ExpressionStep step) { auto* subexpression = program_builder_.GetSubexpression(&node); if (subexpression == nullptr) { diff --git a/eval/compiler/flat_expr_builder_extensions.h b/eval/compiler/flat_expr_builder_extensions.h index 21e37b2a8..0112356e4 100644 --- a/eval/compiler/flat_expr_builder_extensions.h +++ b/eval/compiler/flat_expr_builder_extensions.h @@ -23,6 +23,7 @@ #define THIRD_PARTY_CEL_CPP_EVAL_COMPILER_FLAT_EXPR_BUILDER_EXTENSIONS_H_ #include +#include #include #include #include @@ -84,11 +85,10 @@ class ProgramBuilder { // Must be tied to a ProgramBuilder to coordinate relationships. class Subexpression { private: - using Element = absl::variant, - Subexpression* absl_nonnull>; + using Element = absl::variant; using TreePlan = std::vector; - using FlattenedPlan = std::vector>; + using FlattenedPlan = ExecutionPath; public: struct RecursiveProgram { @@ -105,7 +105,7 @@ class ProgramBuilder { Subexpression& operator=(Subexpression&&) = delete; // Add a program step at the current end of the subexpression. - bool AddStep(std::unique_ptr step) { + bool AddStep(ExpressionStep step) { if (IsRecursive()) { return false; } @@ -141,13 +141,12 @@ class ProgramBuilder { // Accessor for program steps. // // Value is undefined if in the expression has not yet been flattened. - std::vector>& flattened_elements() { + ExecutionPath& flattened_elements() { ABSL_DCHECK(IsFlattened()); return absl::get(program_); } - const std::vector>& - flattened_elements() const { + const ExecutionPath& flattened_elements() const { ABSL_DCHECK(IsFlattened()); return absl::get(program_); } @@ -202,7 +201,7 @@ class ProgramBuilder { // ownership of the given steps. // // Returns false if the subexpression is not currently flattened. - bool ExtractTo(std::vector>& out); + bool ExtractTo(ExecutionPath& out); private: Subexpression(const cel::Expr* self, ProgramBuilder* owner); @@ -287,13 +286,12 @@ class ProgramBuilder { // Note: If successful, the pointer should remain valid until the parent // expression is finalized. Optimizers may modify the program plan which may // free the step at that point. - ExpressionStep* absl_nullable AddStep(std::unique_ptr step); + ExpressionStep* absl_nullable AddStep(ExpressionStep step); void Reset(); private: - static std::vector> - FlattenSubexpression(Subexpression* absl_nonnull expr); + static ExecutionPath FlattenSubexpression(Subexpression* absl_nonnull expr); Subexpression* absl_nullable MakeSubexpression(const cel::Expr* expr); @@ -385,8 +383,13 @@ class PlannerContext { int depth); // Extend the current subplan with the given expression step. + absl::Status AddSubplanStep(const cel::Expr& node, ExpressionStep step); absl::Status AddSubplanStep(const cel::Expr& node, - std::unique_ptr step); + std::unique_ptr step, + int64_t expr_id = -1) { + return AddSubplanStep( + node, ExpressionStep::MakeGenericStep(std::move(step), expr_id)); + } const Resolver& resolver() const { return resolver_; } const cel::TypeReflector& type_reflector() const { return type_reflector_; } diff --git a/eval/compiler/flat_expr_builder_extensions_test.cc b/eval/compiler/flat_expr_builder_extensions_test.cc index 45913e61b..a1e8ecd4b 100644 --- a/eval/compiler/flat_expr_builder_extensions_test.cc +++ b/eval/compiler/flat_expr_builder_extensions_test.cc @@ -78,13 +78,13 @@ class PlannerContextTest : public testing::Test { MATCHER_P(UniquePtrHolds, ptr, "") { const auto& got = arg; - return ptr == got.get(); + return got.IsGenericStep() && ptr == got.GetGenericStep(); } struct SimpleTreeSteps { - const ExpressionStep* a; - const ExpressionStep* b; - const ExpressionStep* c; + const ExpressionStepLogic* a; + const ExpressionStepLogic* b; + const ExpressionStepLogic* c; }; // simulate a program of: @@ -94,20 +94,23 @@ struct SimpleTreeSteps { absl::StatusOr InitSimpleTree( const Expr& a, const Expr& b, const Expr& c, ProgramBuilder& program_builder) { - CEL_ASSIGN_OR_RETURN(auto a_step, CreateConstValueStep(cel::NullValue(), -1)); - CEL_ASSIGN_OR_RETURN(auto b_step, CreateConstValueStep(cel::NullValue(), -1)); - CEL_ASSIGN_OR_RETURN(auto c_step, CreateConstValueStep(cel::NullValue(), -1)); + auto a_step = CreateConstValueStep(cel::NullValue()); + auto b_step = CreateConstValueStep(cel::NullValue()); + auto c_step = CreateConstValueStep(cel::NullValue()); SimpleTreeSteps result{a_step.get(), b_step.get(), c_step.get()}; program_builder.EnterSubexpression(&a); program_builder.EnterSubexpression(&b); - program_builder.AddStep(std::move(b_step)); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(b_step), -1)); program_builder.ExitSubexpression(&b); program_builder.EnterSubexpression(&c); - program_builder.AddStep(std::move(c_step)); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(c_step), -1)); program_builder.ExitSubexpression(&c); - program_builder.AddStep(std::move(a_step)); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(a_step), -1)); program_builder.ExitSubexpression(&a); return result; @@ -160,10 +163,9 @@ TEST_F(PlannerContextTest, ReplacePlan) { ExecutionPath new_a; - ASSERT_OK_AND_ASSIGN(auto new_a_step, - CreateConstValueStep(cel::NullValue(), -1)); - const ExpressionStep* new_a_step_ptr = new_a_step.get(); - new_a.push_back(std::move(new_a_step)); + auto new_a_step = CreateConstValueStep(cel::NullValue()); + const ExpressionStepLogic* new_a_step_ptr = new_a_step.get(); + new_a.push_back(ExpressionStep::MakeGenericStep(std::move(new_a_step), -1)); ASSERT_THAT(context.ReplaceSubplan(a, std::move(new_a)), IsOk()); @@ -251,14 +253,12 @@ TEST_F(PlannerContextTest, ReplacePlanUpdatesSibling) { ExecutionPath new_b; - ASSERT_OK_AND_ASSIGN(auto b1_step, - CreateConstValueStep(cel::NullValue(), -1)); - const ExpressionStep* b1_step_ptr = b1_step.get(); - new_b.push_back(std::move(b1_step)); - ASSERT_OK_AND_ASSIGN(auto b2_step, - CreateConstValueStep(cel::NullValue(), -1)); - const ExpressionStep* b2_step_ptr = b2_step.get(); - new_b.push_back(std::move(b2_step)); + auto b1_step = CreateConstValueStep(cel::NullValue()); + const ExpressionStepLogic* b1_step_ptr = b1_step.get(); + new_b.push_back(ExpressionStep::MakeGenericStep(std::move(b1_step), -1)); + auto b2_step = CreateConstValueStep(cel::NullValue()); + const ExpressionStepLogic* b2_step_ptr = b2_step.get(); + new_b.push_back(ExpressionStep::MakeGenericStep(std::move(b2_step), -1)); ASSERT_THAT(context.ReplaceSubplan(b, std::move(new_b)), IsOk()); @@ -302,10 +302,9 @@ TEST_F(PlannerContextTest, AddSubplanStep) { ASSERT_OK_AND_ASSIGN(auto plan_steps, InitSimpleTree(a, b, c, program_builder)); - ASSERT_OK_AND_ASSIGN(auto b2_step, - CreateConstValueStep(cel::NullValue(), -1)); + auto b2_step = CreateConstValueStep(cel::NullValue()); - const ExpressionStep* b2_step_ptr = b2_step.get(); + const ExpressionStepLogic* b2_step_ptr = b2_step.get(); std::shared_ptr arena; PlannerContext context(env_, resolver_, options_, @@ -332,8 +331,7 @@ TEST_F(PlannerContextTest, AddSubplanStepFailsOnUnknownNode) { ASSERT_THAT(InitSimpleTree(a, b, c, program_builder).status(), IsOk()); - ASSERT_OK_AND_ASSIGN(auto b2_step, - CreateConstValueStep(cel::NullValue(), -1)); + auto b2_step = CreateConstValueStep(cel::NullValue()); std::shared_ptr arena; PlannerContext context(env_, resolver_, options_, @@ -480,8 +478,9 @@ TEST_F(ProgramBuilderTest, ExtractWorks) { program_builder.EnterSubexpression(&b); program_builder.ExitSubexpression(&b); - ASSERT_OK_AND_ASSIGN(auto a_step, CreateConstValueStep(cel::NullValue(), -1)); - program_builder.AddStep(std::move(a_step)); + auto a_step = CreateConstValueStep(cel::NullValue()); + program_builder.AddStep( + ExpressionStep::MakeGenericStep(std::move(a_step), -1)); program_builder.EnterSubexpression(&c); program_builder.ExitSubexpression(&c); program_builder.ExitSubexpression(&a); @@ -563,8 +562,9 @@ TEST_F(ProgramBuilderTest, Recursive) { auto path = program_builder.FlattenMain(); ASSERT_THAT(path, testing::SizeIs(1)); - EXPECT_TRUE(path[0]->GetNativeTypeId() == - cel::NativeTypeId::For()); + EXPECT_TRUE(path[0].IsGenericStep() && + path[0].GetGenericStep()->GetNativeTypeId() == + cel::NativeTypeId::For()); } } // namespace diff --git a/eval/compiler/instrumentation.cc b/eval/compiler/instrumentation.cc index 3e37bdb45..ea3e7ddbb 100644 --- a/eval/compiler/instrumentation.cc +++ b/eval/compiler/instrumentation.cc @@ -69,7 +69,8 @@ class InstrumentOptimizer : public ProgramOptimizer { } return context.AddSubplanStep( - node, std::make_unique(node.id(), instrumentation_)); + node, std::make_unique(node.id(), instrumentation_), + node.id()); } private: diff --git a/eval/compiler/regex_precompilation_optimization.cc b/eval/compiler/regex_precompilation_optimization.cc index 38ef842b9..46c8817af 100644 --- a/eval/compiler/regex_precompilation_optimization.cc +++ b/eval/compiler/regex_precompilation_optimization.cc @@ -194,11 +194,12 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { } else { // otherwise stack-machine program. ExecutionPathView re_plan = context.GetSubplan(re_expr); - if (re_plan.size() == 1 && - re_plan[0]->GetNativeTypeId() == + if (re_plan.size() == 1 && re_plan[0].IsGenericStep() && + re_plan[0].GetGenericStep()->GetNativeTypeId() == NativeTypeId::For()) { constant = - down_cast(re_plan[0].get())->value(); + down_cast(re_plan[0].GetGenericStep()) + ->value(); } } @@ -251,9 +252,10 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { CEL_ASSIGN_OR_RETURN(ExecutionPath new_plan, context.ExtractSubplan(subject)); - CEL_ASSIGN_OR_RETURN( - new_plan.emplace_back(), - CreateRegexMatchStep(std::move(regex_program), call.id())); + CEL_ASSIGN_OR_RETURN(auto step, + CreateRegexMatchStep(std::move(regex_program))); + new_plan.push_back( + ExpressionStep::MakeGenericStep(std::move(step), call.id())); return context.ReplaceSubplan(call, std::move(new_plan)); } diff --git a/eval/eval/cel_expression_flat_impl.cc b/eval/eval/cel_expression_flat_impl.cc index 9e35b41ad..8c78d21ce 100644 --- a/eval/eval/cel_expression_flat_impl.cc +++ b/eval/eval/cel_expression_flat_impl.cc @@ -108,9 +108,12 @@ absl::StatusOr> CelExpressionRecursiveImpl::Create( absl_nonnull std::shared_ptr env, FlatExpression flat_expr) { - if (flat_expr.path().empty() || - flat_expr.path().front()->GetNativeTypeId() != - cel::NativeTypeId::For()) { + const ExpressionStepLogic* logic = nullptr; + if (!flat_expr.path().empty() && flat_expr.path()[0].IsGenericStep()) { + logic = flat_expr.path()[0].GetGenericStep(); + } + if (logic != nullptr && + logic->GetNativeTypeId() != cel::NativeTypeId::For()) { return absl::InvalidArgumentError(absl::StrCat( "Expected a recursive program step", flat_expr.path().size())); } diff --git a/eval/eval/cel_expression_flat_impl.h b/eval/eval/cel_expression_flat_impl.h index 7faf6856a..3590dc788 100644 --- a/eval/eval/cel_expression_flat_impl.h +++ b/eval/eval/cel_expression_flat_impl.h @@ -162,7 +162,7 @@ class CelExpressionRecursiveImpl : public CelExpression { : env_(std::move(env)), flat_expression_(std::move(flat_expression)), root_(cel::internal::down_cast( - flat_expression_.path()[0].get()) + flat_expression_.path()[0].GetGenericStep()) ->wrapped()) {} absl_nonnull std::shared_ptr env_; diff --git a/eval/eval/compiler_constant_step.h b/eval/eval/compiler_constant_step.h index bd514a036..967f6512f 100644 --- a/eval/eval/compiler_constant_step.h +++ b/eval/eval/compiler_constant_step.h @@ -56,8 +56,7 @@ class DirectCompilerConstantStep : public DirectExpressionStep { // inspect the underlying value. class CompilerConstantStep : public ExpressionStepBase { public: - CompilerConstantStep(cel::Value value, int64_t expr_id, bool comes_from_ast) - : ExpressionStepBase(expr_id, comes_from_ast), value_(std::move(value)) {} + explicit CompilerConstantStep(cel::Value value) : value_(std::move(value)) {} absl::Status Evaluate(ExecutionFrame* frame) const override; diff --git a/eval/eval/compiler_constant_step_test.cc b/eval/eval/compiler_constant_step_test.cc index 856ca30e0..8fb226813 100644 --- a/eval/eval/compiler_constant_step_test.cc +++ b/eval/eval/compiler_constant_step_test.cc @@ -47,8 +47,8 @@ class CompilerConstantStepTest : public testing::Test { TEST_F(CompilerConstantStepTest, Evaluate) { ExecutionPath path; - path.push_back( - std::make_unique(cel::IntValue(42), -1, false)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(cel::IntValue(42)))); ExecutionFrame frame(path, empty_activation_, options_, state_); @@ -58,15 +58,15 @@ TEST_F(CompilerConstantStepTest, Evaluate) { } TEST_F(CompilerConstantStepTest, TypeId) { - CompilerConstantStep step(cel::IntValue(42), -1, false); + CompilerConstantStep step(cel::IntValue(42)); - ExpressionStep& abstract_step = step; + ExpressionStepLogic& abstract_step = step; EXPECT_EQ(abstract_step.GetNativeTypeId(), cel::NativeTypeId::For()); } TEST_F(CompilerConstantStepTest, Value) { - CompilerConstantStep step(cel::IntValue(42), -1, false); + CompilerConstantStep step(cel::IntValue(42)); EXPECT_EQ(step.value().GetInt().NativeValue(), 42); } diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc index 81e9c0270..7f6d5b4e2 100644 --- a/eval/eval/comprehension_step.cc +++ b/eval/eval/comprehension_step.cc @@ -60,8 +60,8 @@ AttributeQualifier AttributeQualifierFromValue(const Value& v) { class ComprehensionFinishStep final : public ExpressionStepBase { public: - ComprehensionFinishStep(size_t accu_slot, int64_t expr_id) - : ExpressionStepBase(expr_id), accu_slot_(accu_slot) {} + explicit ComprehensionFinishStep(size_t accu_slot) + : ExpressionStepBase(), accu_slot_(accu_slot) {} absl::Status Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(2)) { @@ -681,9 +681,9 @@ std::unique_ptr CreateDirectComprehensionStep( shortcircuiting, expr_id); } -std::unique_ptr CreateComprehensionFinishStep(size_t accu_slot, - int64_t expr_id) { - return std::make_unique(accu_slot, expr_id); +std::unique_ptr CreateComprehensionFinishStep( + size_t accu_slot) { + return std::make_unique(accu_slot); } } // namespace google::api::expr::runtime diff --git a/eval/eval/comprehension_step.h b/eval/eval/comprehension_step.h index 34a6afc19..f16b62e13 100644 --- a/eval/eval/comprehension_step.h +++ b/eval/eval/comprehension_step.h @@ -27,8 +27,7 @@ namespace google::api::expr::runtime { class ComprehensionInitStep final : public ExpressionStepBase { public: - explicit ComprehensionInitStep(int64_t expr_id) - : ExpressionStepBase(expr_id, /*comes_from_ast=*/false) {} + ComprehensionInitStep() : ExpressionStepBase() {} void set_error_jump_offset(int offset) { error_jump_offset_ = offset; } @@ -40,9 +39,8 @@ class ComprehensionInitStep final : public ExpressionStepBase { class ComprehensionNextStep final : public ExpressionStepBase { public: - ComprehensionNextStep(size_t iter_slot, size_t iter2_slot, size_t accu_slot, - int64_t expr_id) - : ExpressionStepBase(expr_id, /*comes_from_ast=*/false), + ComprehensionNextStep(size_t iter_slot, size_t iter2_slot, size_t accu_slot) + : ExpressionStepBase(), iter_slot_(iter_slot), iter2_slot_(iter2_slot), accu_slot_(accu_slot) {} @@ -70,8 +68,8 @@ class ComprehensionNextStep final : public ExpressionStepBase { class ComprehensionCondStep final : public ExpressionStepBase { public: ComprehensionCondStep(size_t iter_slot, size_t iter2_slot, size_t accu_slot, - bool shortcircuiting, int64_t expr_id) - : ExpressionStepBase(expr_id, /*comes_from_ast=*/false), + bool shortcircuiting) + : ExpressionStepBase(), iter_slot_(iter_slot), iter2_slot_(iter2_slot), accu_slot_(accu_slot), @@ -111,8 +109,8 @@ std::unique_ptr CreateDirectComprehensionStep( // Creates a cleanup step for the comprehension. // Removes the comprehension context then pushes the 'result' sub expression to // the top of the stack. -std::unique_ptr CreateComprehensionFinishStep(size_t accu_slot, - int64_t expr_id); +std::unique_ptr CreateComprehensionFinishStep( + size_t accu_slot); } // namespace google::api::expr::runtime diff --git a/eval/eval/comprehension_step_test.cc b/eval/eval/comprehension_step_test.cc index 681f8af4f..ef01b034d 100644 --- a/eval/eval/comprehension_step_test.cc +++ b/eval/eval/comprehension_step_test.cc @@ -40,6 +40,7 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::BoolValue; using ::cel::Expr; @@ -86,7 +87,7 @@ class ListKeysStepTest : public testing::Test { class GetListKeysResultStep : public ExpressionStepBase { public: - GetListKeysResultStep() : ExpressionStepBase(-1, false) {} + GetListKeysResultStep() : ExpressionStepBase() {} absl::Status Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Pop(1); @@ -102,13 +103,12 @@ MATCHER_P(CelStringValue, val, "") { TEST_F(ListKeysStepTest, MapPartiallyUnknown) { ExecutionPath path; - auto result = CreateIdentStep("var", 0); - ASSERT_OK(result); - path.push_back(*std::move(result)); - ComprehensionInitStep* init_step = new ComprehensionInitStep(1); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); + auto init_step = std::make_unique(); init_step->set_error_jump_offset(1); - path.push_back(absl::WrapUnique(init_step)); - path.push_back(std::make_unique()); + path.push_back(ExpressionStep::MakeGenericStep(std::move(init_step))); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); auto expression = MakeExpression(std::move(path), /*unknown_attributes=*/true); @@ -129,7 +129,7 @@ TEST_F(ListKeysStepTest, MapPartiallyUnknown) { auto eval_result = expression->Evaluate(activation, &arena); - ASSERT_OK(eval_result); + ASSERT_THAT(eval_result, IsOk()); ASSERT_TRUE(eval_result->IsUnknownSet()); const auto& attrs = eval_result->UnknownSetOrDie()->unknown_attributes(); @@ -140,13 +140,12 @@ TEST_F(ListKeysStepTest, MapPartiallyUnknown) { TEST_F(ListKeysStepTest, ErrorPassedThrough) { ExecutionPath path; - auto result = CreateIdentStep("var", 0); - ASSERT_OK(result); - path.push_back(*std::move(result)); - ComprehensionInitStep* init_step = new ComprehensionInitStep(1); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); + auto init_step = std::make_unique(); init_step->set_error_jump_offset(1); - path.push_back(absl::WrapUnique(init_step)); - path.push_back(std::make_unique()); + path.push_back(ExpressionStep::MakeGenericStep(std::move(init_step))); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); auto expression = MakeExpression(std::move(path)); @@ -156,7 +155,7 @@ TEST_F(ListKeysStepTest, ErrorPassedThrough) { // Var not in activation, turns into cel error at eval time. auto eval_result = expression->Evaluate(activation, &arena); - ASSERT_OK(eval_result); + ASSERT_THAT(eval_result, IsOk()); ASSERT_TRUE(eval_result->IsError()); EXPECT_THAT(eval_result->ErrorOrDie()->message(), testing::HasSubstr("\"var\"")); @@ -165,13 +164,12 @@ TEST_F(ListKeysStepTest, ErrorPassedThrough) { TEST_F(ListKeysStepTest, UnknownSetPassedThrough) { ExecutionPath path; - auto result = CreateIdentStep("var", 0); - ASSERT_OK(result); - path.push_back(*std::move(result)); - ComprehensionInitStep* init_step = new ComprehensionInitStep(1); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); + auto init_step = std::make_unique(); init_step->set_error_jump_offset(1); - path.push_back(absl::WrapUnique(init_step)); - path.push_back(std::make_unique()); + path.push_back(ExpressionStep::MakeGenericStep(std::move(init_step))); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); auto expression = MakeExpression(std::move(path), /*unknown_attributes=*/true); @@ -183,7 +181,7 @@ TEST_F(ListKeysStepTest, UnknownSetPassedThrough) { auto eval_result = expression->Evaluate(activation, &arena); - ASSERT_OK(eval_result); + ASSERT_THAT(eval_result, IsOk()); ASSERT_TRUE(eval_result->IsUnknownSet()); EXPECT_THAT(eval_result->UnknownSetOrDie()->unknown_attributes(), SizeIs(1)); } @@ -412,7 +410,7 @@ TEST_F(DirectComprehensionTest, Shortcircuit) { Value result; AttributeTrail trail; - ASSERT_OK(compre_step->Evaluate(frame, result, trail)); + ASSERT_THAT(compre_step->Evaluate(frame, result, trail), IsOk()); EXPECT_THAT(result, BoolValueIs(false)); } @@ -484,7 +482,7 @@ TEST_F(DirectComprehensionTest, Exhaustive) { Value result; AttributeTrail trail; - ASSERT_OK(compre_step->Evaluate(frame, result, trail)); + ASSERT_THAT(compre_step->Evaluate(frame, result, trail), IsOk()); EXPECT_THAT(result, BoolValueIs(false)); } diff --git a/eval/eval/const_value_step.h b/eval/eval/const_value_step.h index c3cf6a424..8a08b580a 100644 --- a/eval/eval/const_value_step.h +++ b/eval/eval/const_value_step.h @@ -20,10 +20,9 @@ inline std::unique_ptr CreateConstValueDirectStep( } // Factory method for Constant AST node expression step. -inline absl::StatusOr> CreateConstValueStep( - cel::Value value, int64_t expr_id, bool comes_from_ast = true) { - return std::make_unique(std::move(value), expr_id, - comes_from_ast); +inline std::unique_ptr CreateConstValueStep( + cel::Value value) { + return std::make_unique(std::move(value)); } } // namespace google::api::expr::runtime diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index 911dbe08f..8ce87ef3f 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -286,9 +286,8 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, // message. class ContainerAccessStep : public ExpressionStepBase { public: - ContainerAccessStep(int64_t expr_id, bool enable_optional_types) - : ExpressionStepBase(expr_id), - enable_optional_types_(enable_optional_types) {} + explicit ContainerAccessStep(bool enable_optional_types) + : ExpressionStepBase(), enable_optional_types_(enable_optional_types) {} absl::Status Evaluate(ExecutionFrame* frame) const override; @@ -367,14 +366,14 @@ std::unique_ptr CreateDirectContainerAccessStep( } // Factory method for Select - based Execution step -absl::StatusOr> CreateContainerAccessStep( - const cel::CallExpr& call, int64_t expr_id, bool enable_optional_types) { +absl::StatusOr> CreateContainerAccessStep( + const cel::CallExpr& call, bool enable_optional_types) { int arg_count = call.args().size() + (call.has_target() ? 1 : 0); if (arg_count != kNumContainerAccessArguments) { return absl::InvalidArgumentError(absl::StrCat( "Invalid argument count for index operation: ", arg_count)); } - return std::make_unique(expr_id, enable_optional_types); + return std::make_unique(enable_optional_types); } } // namespace google::api::expr::runtime diff --git a/eval/eval/container_access_step.h b/eval/eval/container_access_step.h index b7af5e895..5d5fdde81 100644 --- a/eval/eval/container_access_step.h +++ b/eval/eval/container_access_step.h @@ -17,9 +17,8 @@ std::unique_ptr CreateDirectContainerAccessStep( int64_t expr_id); // Factory method for Select - based Execution step -absl::StatusOr> CreateContainerAccessStep( - const cel::CallExpr& call, int64_t expr_id, - bool enable_optional_types = false); +absl::StatusOr> CreateContainerAccessStep( + const cel::CallExpr& call, bool enable_optional_types = false); } // namespace google::api::expr::runtime diff --git a/eval/eval/container_access_step_test.cc b/eval/eval/container_access_step_test.cc index 25bf72223..a0a69b2ef 100644 --- a/eval/eval/container_access_step_test.cc +++ b/eval/eval/container_access_step_test.cc @@ -39,6 +39,7 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::Expr; using ::cel::SourceInfo; @@ -75,15 +76,18 @@ CelValue EvaluateAttributeHelper( key_expr.mutable_ident_expr().set_name("key"); if (use_recursive_impl) { - path.push_back(std::make_unique( - CreateDirectContainerAccessStep(CreateDirectIdentStep("container", 1), - CreateDirectIdentStep("key", 2), - /*enable_optional_types=*/false, 3), + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(CreateDirectContainerAccessStep( + CreateDirectIdentStep("container", 1), + CreateDirectIdentStep("key", 2), + /*enable_optional_types=*/false, 3)), 3)); } else { - path.push_back(std::move(CreateIdentStep("container", 1).value())); - path.push_back(std::move(CreateIdentStep("key", 2).value())); - path.push_back(std::move(CreateContainerAccessStep(call, 3).value())); + path.push_back( + ExpressionStep::MakeGenericStep(CreateIdentStep("container"), 1)); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("key"), 2)); + path.push_back(ExpressionStep::MakeGenericStep( + std::move(CreateContainerAccessStep(call).value()), 3)); } cel::RuntimeOptions options; @@ -227,8 +231,9 @@ TEST_P(ContainerAccessStepUniformityTest, TestMapKeyAccess) { TEST_P(ContainerAccessStepUniformityTest, TestBoolKeyType) { CelMapBuilder cel_map; - ASSERT_OK(cel_map.Add(CelValue::CreateBool(true), - CelValue::CreateStringView("value_true"))); + ASSERT_THAT(cel_map.Add(CelValue::CreateBool(true), + CelValue::CreateStringView("value_true")), + IsOk()); CelValue result = EvaluateAttribute(CelValue::CreateMap(&cel_map), CelValue::CreateBool(true), @@ -267,7 +272,7 @@ TEST_F(ContainerAccessStepTest, TestInvalidReceiverCreateContainerAccessStep) { Expr& extra_arg = call.mutable_args().emplace_back(); extra_arg.mutable_const_expr().set_bool_value(true); - EXPECT_THAT(CreateContainerAccessStep(call, 0).status(), + EXPECT_THAT(CreateContainerAccessStep(call).status(), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Invalid argument count"))); } @@ -285,7 +290,7 @@ TEST_F(ContainerAccessStepTest, TestInvalidGlobalCreateContainerAccessStep) { Expr& extra_arg = call.mutable_args().emplace_back(); extra_arg.mutable_const_expr().set_bool_value(true); - EXPECT_THAT(CreateContainerAccessStep(call, 0).status(), + EXPECT_THAT(CreateContainerAccessStep(call).status(), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Invalid argument count"))); } diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index bb977ce94..54c9ee8ba 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -35,11 +35,8 @@ using ::cel::common_internal::NewListValueBuilder; class CreateListStep : public ExpressionStepBase { public: - CreateListStep(int64_t expr_id, int list_size, - absl::flat_hash_set optional_indices) - : ExpressionStepBase(expr_id), - list_size_(list_size), - optional_indices_(std::move(optional_indices)) {} + CreateListStep(int list_size, absl::flat_hash_set optional_indices) + : list_size_(list_size), optional_indices_(std::move(optional_indices)) {} absl::Status Evaluate(ExecutionFrame* frame) const override; @@ -226,7 +223,7 @@ class CreateListDirectStep : public DirectExpressionStep { class MutableListStep : public ExpressionStepBase { public: - explicit MutableListStep(int64_t expr_id) : ExpressionStepBase(expr_id) {} + MutableListStep() = default; absl::Status Evaluate(ExecutionFrame* frame) const override; }; @@ -247,9 +244,9 @@ class DirectMutableListStep : public DirectExpressionStep { AttributeTrail& attribute) const override; }; -absl::Status DirectMutableListStep::Evaluate( - ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { +absl::Status DirectMutableListStep::Evaluate(ExecutionFrameBase& frame, + Value& result, + AttributeTrail& attribute) const { result = cel::CustomListValue( cel::common_internal::NewMutableListValue(frame.arena()), frame.arena()); return absl::OkStatus(); @@ -264,15 +261,15 @@ std::unique_ptr CreateDirectListStep( std::move(deps), std::move(optional_indices), expr_id); } -absl::StatusOr> CreateCreateListStep( - const cel::ListExpr& create_list_expr, int64_t expr_id) { +absl::StatusOr> CreateCreateListStep( + const cel::ListExpr& create_list_expr) { return std::make_unique( - expr_id, create_list_expr.elements().size(), + create_list_expr.elements().size(), MakeOptionalIndicesSet(create_list_expr)); } -std::unique_ptr CreateMutableListStep(int64_t expr_id) { - return std::make_unique(expr_id); +std::unique_ptr CreateMutableListStep() { + return std::make_unique(); } std::unique_ptr CreateDirectMutableListStep( diff --git a/eval/eval/create_list_step.h b/eval/eval/create_list_step.h index b60a5e9c8..c377338c5 100644 --- a/eval/eval/create_list_step.h +++ b/eval/eval/create_list_step.h @@ -19,14 +19,14 @@ std::unique_ptr CreateDirectListStep( absl::flat_hash_set optional_indices, int64_t expr_id); // Factory method for CreateList which constructs an immutable list. -absl::StatusOr> CreateCreateListStep( - const cel::ListExpr& create_list_expr, int64_t expr_id); +absl::StatusOr> CreateCreateListStep( + const cel::ListExpr& create_list_expr); // Factory method for CreateList which constructs a mutable list. // // This is intended for the list construction step is generated for a // list-building comprehension (rather than a user authored expression). -std::unique_ptr CreateMutableListStep(int64_t expr_id); +std::unique_ptr CreateMutableListStep(); // Factory method for CreateList which constructs a mutable list. // diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index dfc9557a6..ebb1ecb41 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -80,16 +80,13 @@ absl::StatusOr RunExpression( for (auto value : values) { auto& expr0 = create_list.mutable_elements().emplace_back().mutable_expr(); expr0.mutable_const_expr().set_int64_value(value); - CEL_ASSIGN_OR_RETURN( - auto const_step, - CreateConstValueStep(cel::interop_internal::CreateIntValue(value), - /*expr_id=*/-1)); - path.push_back(std::move(const_step)); + path.push_back(ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::interop_internal::CreateIntValue(value)))); } - CEL_ASSIGN_OR_RETURN(auto step, - CreateCreateListStep(create_list, dummy_expr.id())); - path.push_back(std::move(step)); + CEL_ASSIGN_OR_RETURN(auto step, CreateCreateListStep(create_list)); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step), dummy_expr.id())); cel::RuntimeOptions options; if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; @@ -122,15 +119,13 @@ absl::StatusOr RunExpressionWithCelValues( expr0.set_id(ind); expr0.mutable_ident_expr().set_name(var_name); - CEL_ASSIGN_OR_RETURN(auto ident_step, - CreateIdentStep(var_name, /*expr_id=*/-1)); - path.push_back(std::move(ident_step)); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep(var_name))); activation.InsertValue(var_name, value); } - CEL_ASSIGN_OR_RETURN(auto step0, - CreateCreateListStep(create_list, dummy_expr.id())); - path.push_back(std::move(step0)); + CEL_ASSIGN_OR_RETURN(auto step0, CreateCreateListStep(create_list)); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step0), dummy_expr.id())); cel::RuntimeOptions options; if (enable_unknowns) { @@ -164,9 +159,9 @@ TEST(CreateListStepTest, TestCreateListStackUnderflow) { auto& expr0 = create_list.mutable_elements().emplace_back().mutable_expr(); expr0.mutable_const_expr().set_int64_value(1); - ASSERT_OK_AND_ASSIGN(auto step0, - CreateCreateListStep(create_list, dummy_expr.id())); - path.push_back(std::move(step0)); + ASSERT_OK_AND_ASSIGN(auto step0, CreateCreateListStep(create_list)); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step0), dummy_expr.id())); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl cel_expr( @@ -292,7 +287,7 @@ TEST(CreateDirectListStep, Basic) { cel::Value result; AttributeTrail attr; - ASSERT_OK(step->Evaluate(frame, result, attr)); + ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).Size(), IsOkAndHolds(2)); @@ -320,7 +315,7 @@ TEST(CreateDirectListStep, ForwardFirstError) { cel::Value result; AttributeTrail attr; - ASSERT_OK(step->Evaluate(frame, result, attr)); + ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).NativeValue(), @@ -332,9 +327,10 @@ std::vector UnknownAttrNames(const UnknownValue& v) { names.reserve(v.ToAttributeSet().size()); for (const auto& attr : v.ToAttributeSet()) { - EXPECT_OK(attr.AsString().status()); + EXPECT_THAT(attr.AsString().status(), IsOk()); names.push_back(attr.AsString().value_or("")); } + return names; } @@ -368,7 +364,7 @@ TEST(CreateDirectListStep, MergeUnknowns) { cel::Value result; AttributeTrail attr; - ASSERT_OK(step->Evaluate(frame, result, attr)); + ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(UnknownAttrNames(Cast(result)), @@ -399,7 +395,7 @@ TEST(CreateDirectListStep, ErrorBeforeUnknown) { cel::Value result; AttributeTrail attr; - ASSERT_OK(step->Evaluate(frame, result, attr)); + ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).NativeValue(), @@ -447,7 +443,7 @@ TEST(CreateDirectListStep, MissingAttribute) { cel::Value result; AttributeTrail attr; - ASSERT_OK(step->Evaluate(frame, result, attr)); + ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT( @@ -476,7 +472,7 @@ TEST(CreateDirectListStep, OptionalPresentSet) { cel::Value result; AttributeTrail attr; - ASSERT_OK(step->Evaluate(frame, result, attr)); + ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); ASSERT_TRUE(InstanceOf(result)); auto list = Cast(result); @@ -509,7 +505,7 @@ TEST(CreateDirectListStep, OptionalAbsentNotSet) { cel::Value result; AttributeTrail attr; - ASSERT_OK(step->Evaluate(frame, result, attr)); + ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); ASSERT_TRUE(InstanceOf(result)); auto list = Cast(result); @@ -542,7 +538,7 @@ TEST(CreateDirectListStep, PartialUnknown) { cel::Value result; AttributeTrail attr; - ASSERT_OK(step->Evaluate(frame, result, attr)); + ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(UnknownAttrNames(Cast(result)), diff --git a/eval/eval/create_map_step.cc b/eval/eval/create_map_step.cc index 451181e75..e90de1fc7 100644 --- a/eval/eval/create_map_step.cc +++ b/eval/eval/create_map_step.cc @@ -52,10 +52,9 @@ using ::cel::common_internal::NewMutableMapValue; // `CreateStruct` implementation for map. class CreateStructStepForMap final : public ExpressionStepBase { public: - CreateStructStepForMap(int64_t expr_id, size_t entry_count, + CreateStructStepForMap(size_t entry_count, absl::flat_hash_set optional_indices) - : ExpressionStepBase(expr_id), - entry_count_(entry_count), + : entry_count_(entry_count), optional_indices_(std::move(optional_indices)) {} absl::Status Evaluate(ExecutionFrame* frame) const override; @@ -235,9 +234,9 @@ absl::Status DirectCreateMapStep::Evaluate( return absl::OkStatus(); } -class MutableMapStep final : public ExpressionStep { +class MutableMapStep final : public ExpressionStepBase { public: - explicit MutableMapStep(int64_t expr_id) : ExpressionStep(expr_id) {} + MutableMapStep() = default; absl::Status Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Push(cel::CustomMapValue( @@ -268,17 +267,16 @@ std::unique_ptr CreateDirectCreateMapStep( std::move(deps), std::move(optional_indices), expr_id); } -absl::StatusOr> CreateCreateStructStepForMap( - size_t entry_count, absl::flat_hash_set optional_indices, - int64_t expr_id) { +absl::StatusOr> +CreateCreateStructStepForMap(size_t entry_count, + absl::flat_hash_set optional_indices) { // Make map-creating step. - return std::make_unique(expr_id, entry_count, + return std::make_unique(entry_count, std::move(optional_indices)); } -absl::StatusOr> CreateMutableMapStep( - int64_t expr_id) { - return std::make_unique(expr_id); +std::unique_ptr CreateMutableMapStep() { + return std::make_unique(); } std::unique_ptr CreateDirectMutableMapStep( diff --git a/eval/eval/create_map_step.h b/eval/eval/create_map_step.h index cf5e94644..599a48534 100644 --- a/eval/eval/create_map_step.h +++ b/eval/eval/create_map_step.h @@ -36,16 +36,15 @@ std::unique_ptr CreateDirectCreateMapStep( absl::flat_hash_set optional_indices, int64_t expr_id); // Creates an `ExpressionStep` which performs `CreateStruct` for a map. -absl::StatusOr> CreateCreateStructStepForMap( - size_t entry_count, absl::flat_hash_set optional_indices, - int64_t expr_id); +absl::StatusOr> +CreateCreateStructStepForMap(size_t entry_count, + absl::flat_hash_set optional_indices); // Factory method for CreateMap which constructs a mutable map. // // This is intended for the map construction step is generated for a // map-building comprehension (rather than a user authored expression). -absl::StatusOr> CreateMutableMapStep( - int64_t expr_id); +std::unique_ptr CreateMutableMapStep(); // Factory method for CreateMap which constructs a mutable map. // diff --git a/eval/eval/create_map_step_test.cc b/eval/eval/create_map_step_test.cc index dbc9adb5a..36a6eebd4 100644 --- a/eval/eval/create_map_step_test.cc +++ b/eval/eval/create_map_step_test.cc @@ -71,14 +71,11 @@ absl::StatusOr CreateStackMachineProgram( std::string key_name = absl::StrCat("key", index); std::string value_name = absl::StrCat("value", index); - CEL_ASSIGN_OR_RETURN(auto step_key, - CreateIdentStep(key_name, /*expr_id=*/-1)); + auto step_key = CreateIdentStep(key_name); + auto step_value = CreateIdentStep(value_name); - CEL_ASSIGN_OR_RETURN(auto step_value, - CreateIdentStep(value_name, /*expr _id=*/-1)); - - path.push_back(std::move(step_key)); - path.push_back(std::move(step_value)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step_key))); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step_value))); activation.InsertValue(key_name, item.first); activation.InsertValue(value_name, item.second); @@ -87,9 +84,9 @@ absl::StatusOr CreateStackMachineProgram( index++; } - CEL_ASSIGN_OR_RETURN( - auto step1, CreateCreateStructStepForMap(values.size(), {}, expr1.id())); - path.push_back(std::move(step1)); + CEL_ASSIGN_OR_RETURN(auto step1, + CreateCreateStructStepForMap(values.size(), {})); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1), expr1.id())); return path; } @@ -113,8 +110,9 @@ absl::StatusOr CreateRecursiveProgram( index++; } - path.push_back(std::make_unique( - CreateDirectCreateMapStep(std::move(deps), {}, -1), -1)); + path.push_back( + ExpressionStep::MakeGenericStep(std::make_unique( + CreateDirectCreateMapStep(std::move(deps), {}, -1)))); return path; } diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index e3c8365a4..a4467a736 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -48,10 +48,9 @@ using ::cel::Value; // `CreateStruct` implementation for message/struct. class CreateStructStepForStruct final : public ExpressionStepBase { public: - CreateStructStepForStruct(int64_t expr_id, std::string name, - std::vector entries, + CreateStructStepForStruct(std::string name, std::vector entries, absl::flat_hash_set optional_indices) - : ExpressionStepBase(expr_id), + : ExpressionStepBase(), name_(std::move(name)), entries_(std::move(entries)), optional_indices_(std::move(optional_indices)) {} @@ -261,12 +260,11 @@ std::unique_ptr CreateDirectCreateStructStep( std::move(optional_indices)); } -std::unique_ptr CreateCreateStructStep( +std::unique_ptr CreateCreateStructStep( std::string name, std::vector field_keys, - absl::flat_hash_set optional_indices, int64_t expr_id) { + absl::flat_hash_set optional_indices) { // MakeOptionalIndicesSet(create_struct_expr) return std::make_unique( - expr_id, std::move(name), std::move(field_keys), - std::move(optional_indices)); + std::move(name), std::move(field_keys), std::move(optional_indices)); } } // namespace google::api::expr::runtime diff --git a/eval/eval/create_struct_step.h b/eval/eval/create_struct_step.h index eb80634f8..11880493a 100644 --- a/eval/eval/create_struct_step.h +++ b/eval/eval/create_struct_step.h @@ -35,9 +35,9 @@ std::unique_ptr CreateDirectCreateStructStep( // Creates an `ExpressionStep` which performs `CreateStruct` for a // message/struct. -std::unique_ptr CreateCreateStructStep( +std::unique_ptr CreateCreateStructStep( std::string name, std::vector field_keys, - absl::flat_hash_set optional_indices, int64_t expr_id); + absl::flat_hash_set optional_indices); } // namespace google::api::expr::runtime diff --git a/eval/eval/create_struct_step_test.cc b/eval/eval/create_struct_step_test.cc index 666dddcdb..056b10267 100644 --- a/eval/eval/create_struct_step_test.cc +++ b/eval/eval/create_struct_step_test.cc @@ -72,16 +72,14 @@ using ::testing::Pointwise; absl::StatusOr MakeStackMachinePath(absl::string_view field) { ExecutionPath path; - CEL_ASSIGN_OR_RETURN(auto step0, CreateIdentStep("message", /*expr_id=*/-1)); + auto step0 = CreateIdentStep("message"); auto step1 = CreateCreateStructStep("google.api.expr.runtime.TestMessage", {std::string(field)}, - /*optional_indices=*/{}, + /*optional_indices=*/{}); - /*id=*/-1); - - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0))); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); return path; } @@ -99,7 +97,8 @@ absl::StatusOr MakeRecursivePath(absl::string_view field) { /*id=*/-1); - path.push_back(std::make_unique(std::move(step1), -1)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(std::move(step1)))); return path; } @@ -212,14 +211,13 @@ TEST_P(CreateCreateStructStepTest, TestEmptyMessageCreation) { /*deps=*/{}, /*optional_indices=*/{}, /*id=*/-1); - path.push_back( - std::make_unique(std::move(step), /*id=*/-1)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(std::move(step)))); } else { auto step = CreateCreateStructStep("google.api.expr.runtime.TestMessage", /*fields=*/{}, - /*optional_indices=*/{}, - /*id=*/-1); - path.push_back(std::move(step)); + /*optional_indices=*/{}); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); } cel::RuntimeOptions options; @@ -279,7 +277,7 @@ TEST(CreateCreateStructStepTest, TestMessageCreateWithUnknown) { auto eval_status = RunExpression(env, "bool_value", CelValue::CreateUnknownSet(&unknown_set), &arena, true, /*enable_recursive_planning=*/false); - ASSERT_OK(eval_status); + ASSERT_THAT(eval_status, IsOk()); ASSERT_TRUE(eval_status->IsUnknownSet()); } @@ -293,7 +291,7 @@ TEST(CreateCreateStructStepTest, TestMessageCreateWithUnknownRecursive) { auto eval_status = RunExpression(env, "bool_value", CelValue::CreateUnknownSet(&unknown_set), &arena, true, /*enable_recursive_planning=*/true); - ASSERT_OK(eval_status); + ASSERT_THAT(eval_status, IsOk()); ASSERT_TRUE(eval_status->IsUnknownSet()) << eval_status->DebugString(); } diff --git a/eval/eval/direct_expression_step.h b/eval/eval/direct_expression_step.h index f11479065..ac783d2d7 100644 --- a/eval/eval/direct_expression_step.h +++ b/eval/eval/direct_expression_step.h @@ -77,10 +77,11 @@ class DirectExpressionStep { }; // Wrapper for direct steps to work with the stack machine impl. -class WrappedDirectStep : public ExpressionStep { +class WrappedDirectStep : public ExpressionStepLogic { public: - WrappedDirectStep(std::unique_ptr impl, int64_t expr_id) - : ExpressionStep(expr_id, false), impl_(std::move(impl)) {} + explicit WrappedDirectStep(std::unique_ptr impl, + int64_t expr_id = -1) + : impl_(std::move(impl)) {} absl::Status Evaluate(ExecutionFrame* frame) const override; diff --git a/eval/eval/equality_steps.cc b/eval/eval/equality_steps.cc index 25bfe2e8a..0069aef1e 100644 --- a/eval/eval/equality_steps.cc +++ b/eval/eval/equality_steps.cc @@ -109,8 +109,7 @@ class DirectEqualityStep : public DirectExpressionStep { class IterativeEqualityStep : public ExpressionStepBase { public: - explicit IterativeEqualityStep(bool negation, int64_t expr_id) - : ExpressionStepBase(expr_id), negation_(negation) {} + explicit IterativeEqualityStep(bool negation) : negation_(negation) {} absl::Status Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(2)) { @@ -248,7 +247,7 @@ class DirectInStep : public DirectExpressionStep { class IterativeInStep : public ExpressionStepBase { public: - explicit IterativeInStep(int64_t expr_id) : ExpressionStepBase(expr_id) {} + IterativeInStep() = default; absl::Status Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(2)) { @@ -276,9 +275,8 @@ std::unique_ptr CreateDirectEqualityStep( } // Factory method for iterative _==_ and _!=_ Execution step -std::unique_ptr CreateEqualityStep(bool negation, - int64_t expr_id) { - return std::make_unique(negation, expr_id); +std::unique_ptr CreateEqualityStep(bool negation) { + return std::make_unique(negation); } // Factory method for recursive @in Execution step @@ -290,8 +288,8 @@ std::unique_ptr CreateDirectInStep( } // Factory method for iterative @in Execution step -std::unique_ptr CreateInStep(int64_t expr_id) { - return std::make_unique(expr_id); +std::unique_ptr CreateInStep() { + return std::make_unique(); } } // namespace google::api::expr::runtime diff --git a/eval/eval/equality_steps.h b/eval/eval/equality_steps.h index eb3bec4ca..7058f9c56 100644 --- a/eval/eval/equality_steps.h +++ b/eval/eval/equality_steps.h @@ -29,8 +29,7 @@ std::unique_ptr CreateDirectEqualityStep( std::unique_ptr rhs, bool negation, int64_t expr_id); // Factory method for iterative _==_/_!=_ Execution step -std::unique_ptr CreateEqualityStep(bool negation, - int64_t expr_id); +std::unique_ptr CreateEqualityStep(bool negation); // Factory method for recursive @in Execution step std::unique_ptr CreateDirectInStep( @@ -38,7 +37,7 @@ std::unique_ptr CreateDirectInStep( std::unique_ptr container, int64_t expr_id); // Factory method for iterative @in Execution step -std::unique_ptr CreateInStep(int64_t expr_id); +std::unique_ptr CreateInStep(); } // namespace google::api::expr::runtime diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index 168ce7603..3b7e93626 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -51,15 +51,15 @@ using ::cel::ValueKind; using ::cel::test::BoolValueIs; using ::cel::test::ValueKindIs; -class ValueStep : public ExpressionStep, public DirectExpressionStep { +class ValueStep : public ExpressionStepLogic, public DirectExpressionStep { public: ValueStep(Value value, Attribute attr) - : ExpressionStep(-1), + : ExpressionStepLogic(), DirectExpressionStep(-1), value_(std::move(value)), attr_(std::move(attr)) {} explicit ValueStep(Value value) - : ExpressionStep(-1), + : ExpressionStepLogic(), DirectExpressionStep(-1), value_(std::move(value)), attr_() {} @@ -148,11 +148,12 @@ TEST(IterativeTest, PartialAttrUnknown) { cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); - std::vector> steps; - steps.push_back( - std::make_unique(IntValue(1), cel::Attribute("foo"))); - steps.push_back(std::make_unique(IntValue(2))); - steps.push_back(CreateEqualityStep(false, -1)); + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(IntValue(1), cel::Attribute("foo")))); + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(IntValue(2)))); + steps.push_back(ExpressionStep::MakeGenericStep(CreateEqualityStep(false))); activation.SetUnknownPatterns({cel::AttributePattern( "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); @@ -178,11 +179,12 @@ TEST(IterativeTest, PartialAttrUnknownDisabled) { cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); - std::vector> steps; - steps.push_back( - std::make_unique(IntValue(1), cel::Attribute("foo"))); - steps.push_back(std::make_unique(IntValue(2))); - steps.push_back(CreateEqualityStep(false, -1)); + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(IntValue(1), cel::Attribute("foo")))); + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(IntValue(2)))); + steps.push_back(ExpressionStep::MakeGenericStep(CreateEqualityStep(false))); activation.SetUnknownPatterns({cel::AttributePattern( "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); @@ -284,12 +286,13 @@ TEST_P(EqualsTest, Iterative) { cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); - std::vector> steps; - steps.push_back( - std::make_unique(MakeValue(test_case.lhs, &arena))); + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeValue(test_case.lhs, &arena)))); + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeValue(test_case.rhs, &arena)))); steps.push_back( - std::make_unique(MakeValue(test_case.rhs, &arena))); - steps.push_back(CreateEqualityStep(test_case.negation, -1)); + ExpressionStep::MakeGenericStep(CreateEqualityStep(test_case.negation))); ExecutionFrame frame(steps, activation, opts, state); @@ -465,12 +468,12 @@ TEST_P(InTest, Iterative) { cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); - std::vector> steps; - steps.push_back( - std::make_unique(MakeValue(test_case.lhs, &arena))); - steps.push_back( - std::make_unique(MakeValue(test_case.rhs, &arena))); - steps.push_back(CreateInStep(-1)); + ExecutionPath steps; + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeValue(test_case.lhs, &arena)))); + steps.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(MakeValue(test_case.rhs, &arena)))); + steps.push_back(ExpressionStep::MakeGenericStep(CreateInStep())); ExecutionFrame frame(steps, activation, opts, state); diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index a7f8e74ea..0aa6000a4 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -15,9 +15,6 @@ #include "eval/eval/evaluator_core.h" #include -#include -#include -#include #include #include "absl/base/nullability.h" @@ -46,7 +43,7 @@ const ExpressionStep* ExecutionFrame::Next() { const size_t end_pos = execution_path_.size(); if (ABSL_PREDICT_TRUE(pc_ < end_pos)) { - const auto* step = execution_path_[pc_++].get(); + const auto* step = &execution_path_[pc_++]; ABSL_ASSUME(step != nullptr); return step; } @@ -134,13 +131,7 @@ absl::StatusOr ExecutionFrame::Evaluate( "Try to disable short-circuiting."; continue; } - const int64_t id = expr->id(); - // Skip if the id is out of range. - // Will take advantage of this in a follow up to bit pack the id. - if (id < 0 || id > std::numeric_limits::max()) { - continue; - } - if (EvaluationStatus status(listener(id, value_stack().Peek(), + if (EvaluationStatus status(listener(expr->id(), value_stack().Peek(), descriptor_pool(), message_factory(), arena())); !status.ok()) { diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index d1d4e5438..f25be8448 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -17,11 +17,14 @@ #include #include +#include #include +#include #include #include #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -54,16 +57,76 @@ class ExecutionFrame; using EvaluationListener = cel::TraceableProgram::EvaluationListener; -// Class Expression represents single execution step. +class ExpressionStepLogic; + +enum class ExpressionStepKind : uint16_t { + kMovedFrom = 0, + kGenericLogic = 1, +}; + class ExpressionStep { public: - explicit ExpressionStep(int64_t id, bool comes_from_ast = true) - : id_(id), comes_from_ast_(comes_from_ast) {} - + // Move-only. ExpressionStep(const ExpressionStep&) = delete; ExpressionStep& operator=(const ExpressionStep&) = delete; + ExpressionStep(ExpressionStep&&); + ExpressionStep& operator=(ExpressionStep&&); + + // Returns corresponding expression object ID. + // Requires that the input expression has IDs assigned to sub-expressions, + // e.g. via a checker. The default value 0 is returned if there is no + // expression associated (e.g. a jump step), or if there is no ID assigned to + // the corresponding expression. Useful for error scenarios where information + // from Expr object is needed to create CelError. + int64_t id() const { + return header_.id >= 0 ? static_cast(header_.id) : -1; + } + + // Returns if the execution step comes from AST. + bool comes_from_ast() const { return header_.id >= 0; } + + absl::Status Evaluate(ExecutionFrame* context) const; + + template + T Get() const; + template + bool Is() const; + + const ExpressionStepLogic* GetGenericStep() const; + bool IsGenericStep() const; + + static ExpressionStep MakeGenericStep( + std::unique_ptr logic, int64_t id = -1) { + if (id < 0 || id > std::numeric_limits::max()) { + id = -1; + } + return ExpressionStep(ExpressionStepKind::kGenericLogic, id, + std::move(logic)); + } + + private: + struct Header { + ExpressionStepKind kind; + uint16_t reserved; + int32_t id; + }; - virtual ~ExpressionStep() = default; + friend void swap(ExpressionStep& lhs, ExpressionStep& rhs); + + ExpressionStep() = default; + + ExpressionStep(ExpressionStepKind kind, int32_t id, + std::unique_ptr logic) + : header_{kind, 0, id}, logic_(std::move(logic)) {} + + Header header_; + std::unique_ptr logic_; +}; + +// Class Expression represents single execution step. +class ExpressionStepLogic { + public: + virtual ~ExpressionStepLogic() = default; // Performs actual evaluation. // Values are passed between Expression objects via EvaluatorStack, which is @@ -74,32 +137,16 @@ class ExpressionStep { // modify execution order(perform jumps). virtual absl::Status Evaluate(ExecutionFrame* context) const = 0; - // Returns corresponding expression object ID. - // Requires that the input expression has IDs assigned to sub-expressions, - // e.g. via a checker. The default value 0 is returned if there is no - // expression associated (e.g. a jump step), or if there is no ID assigned to - // the corresponding expression. Useful for error scenarios where information - // from Expr object is needed to create CelError. - int64_t id() const { return id_; } - - // Returns if the execution step comes from AST. - bool comes_from_ast() const { return comes_from_ast_; } - // Return the type of the underlying expression step for special handling in // the planning phase. This should only be overridden by special cases, and // callers must not make any assumptions about the default case. virtual cel::NativeTypeId GetNativeTypeId() const { return cel::NativeTypeId(); } - - private: - const int64_t id_; - const bool comes_from_ast_; }; -using ExecutionPath = std::vector>; -using ExecutionPathView = - absl::Span>; +using ExecutionPath = std::vector; +using ExecutionPathView = absl::Span; // Class that wraps the state that needs to be allocated for expression // evaluation. This can be reused to save on allocations. @@ -397,7 +444,7 @@ class ExecutionFrame : public ExecutionFrameBase { void Call(size_t slot_index, size_t subexpression_index) { ABSL_DCHECK_LT(subexpression_index, subexpressions_.size()); ExecutionPathView subexpression = subexpressions_[subexpression_index]; - ABSL_DCHECK(subexpression != execution_path_); + ABSL_DCHECK(subexpression.data() != execution_path_.data()); size_t return_pc = pc_; // return pc == size() is supported (a tail call). ABSL_DCHECK_LE(return_pc, execution_path_.size()); @@ -533,6 +580,61 @@ class FlatExpression { absl_nullable std::shared_ptr arena_; }; +// Implementation details. + +inline absl::Status ExpressionStep::Evaluate(ExecutionFrame* context) const { + ABSL_DCHECK_EQ(header_.kind, ExpressionStepKind::kGenericLogic); + return logic_->Evaluate(context); +} + +template +inline T ExpressionStep::Get() const { + if constexpr (std::is_same_v) { + return GetGenericStep(); + } else { + static_assert(sizeof(T) == 0, "unsupported type"); + } + ABSL_UNREACHABLE(); +} + +inline const ExpressionStepLogic* ExpressionStep::GetGenericStep() const { + ABSL_DCHECK_EQ(header_.kind, ExpressionStepKind::kGenericLogic); + return logic_.get(); +} + +template +inline bool ExpressionStep::Is() const { + if constexpr (std::is_same_v) { + return header_.kind == ExpressionStepKind::kGenericLogic; + } else { + static_assert(sizeof(T) == 0, "unsupported type"); + } + ABSL_UNREACHABLE(); +} + +inline bool ExpressionStep::IsGenericStep() const { + return header_.kind == ExpressionStepKind::kGenericLogic; +} + +inline ExpressionStep::ExpressionStep(ExpressionStep&& other) + : ExpressionStep() { + using std::swap; + swap(*this, other); +} + +inline ExpressionStep& ExpressionStep::operator=(ExpressionStep&& other) { + using std::swap; + swap(*this, other); + return *this; +} + +inline void swap(ExpressionStep& lhs, ExpressionStep& rhs) { + using std::swap; + + swap(lhs.header_, rhs.header_); + swap(lhs.logic_, rhs.logic_); +} + } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_EVALUATOR_CORE_H_ diff --git a/eval/eval/evaluator_core_test.cc b/eval/eval/evaluator_core_test.cc index 8d61c4659..873bc7365 100644 --- a/eval/eval/evaluator_core_test.cc +++ b/eval/eval/evaluator_core_test.cc @@ -1,8 +1,10 @@ #include "eval/eval/evaluator_core.h" #include +#include #include #include +#include #include "cel/expr/syntax.pb.h" #include "absl/status/status.h" @@ -25,6 +27,7 @@ namespace google::api::expr::runtime { +using ::absl_testing::IsOk; using ::cel::IntValue; using ::cel::TypeProvider; using ::cel::interop_internal::CreateIntValue; @@ -32,13 +35,14 @@ using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::Expr; using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::testing::_; +using ::testing::ElementsAre; using ::testing::Eq; // Fake expression implementation // Pushes int64(0) on top of value stack. -class FakeConstExpressionStep : public ExpressionStep { +class FakeConstExpressionStep : public ExpressionStepLogic { public: - FakeConstExpressionStep() : ExpressionStep(0, true) {} + FakeConstExpressionStep() = default; absl::Status Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Push(CreateIntValue(0)); @@ -48,9 +52,9 @@ class FakeConstExpressionStep : public ExpressionStep { // Fake expression implementation // Increments argument on top of the stack. -class FakeIncrementExpressionStep : public ExpressionStep { +class FakeIncrementExpressionStep : public ExpressionStepLogic { public: - FakeIncrementExpressionStep() : ExpressionStep(0, true) {} + FakeIncrementExpressionStep() = default; absl::Status Evaluate(ExecutionFrame* frame) const override { auto value = frame->value_stack().Peek(); @@ -67,15 +71,13 @@ TEST(EvaluatorCoreTest, ExecutionFrameNext) { google::protobuf::Arena arena; cel::runtime_internal::RuntimeTypeProvider type_provider( cel::internal::GetTestingDescriptorPool()); - auto const_step = std::make_unique(); - auto incr_step1 = std::make_unique(); - auto incr_step2 = std::make_unique(); - path.push_back(std::move(const_step)); - path.push_back(std::move(incr_step1)); - path.push_back(std::move(incr_step2)); - - auto dummy_expr = std::make_unique(); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kDisabled; @@ -87,21 +89,20 @@ TEST(EvaluatorCoreTest, ExecutionFrameNext) { cel::internal::GetTestingMessageFactory(), &arena); ExecutionFrame frame(path, activation, options, state); - EXPECT_THAT(frame.Next(), Eq(path[0].get())); - EXPECT_THAT(frame.Next(), Eq(path[1].get())); - EXPECT_THAT(frame.Next(), Eq(path[2].get())); + EXPECT_THAT(frame.Next(), Eq(&path[0])); + EXPECT_THAT(frame.Next(), Eq(&path[1])); + EXPECT_THAT(frame.Next(), Eq(&path[2])); EXPECT_THAT(frame.Next(), Eq(nullptr)); } TEST(EvaluatorCoreTest, SimpleEvaluatorTest) { ExecutionPath path; - auto const_step = std::make_unique(); - auto incr_step1 = std::make_unique(); - auto incr_step2 = std::make_unique(); - - path.push_back(std::move(const_step)); - path.push_back(std::move(incr_step1)); - path.push_back(std::move(incr_step2)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique())); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( @@ -113,7 +114,7 @@ TEST(EvaluatorCoreTest, SimpleEvaluatorTest) { google::protobuf::Arena arena; auto status = impl.Evaluate(activation, &arena); - EXPECT_OK(status); + ASSERT_THAT(status, IsOk()); auto value = status.value(); EXPECT_TRUE(value.IsInt64()); @@ -126,6 +127,44 @@ class MockTraceCallback { (int64_t expr_id, const CelValue& value, google::protobuf::Arena*)); }; +TEST(EvaluatorCoreTest, TraceFilterById) { + ExecutionPath path; + // Step with ID 0 should trigger trace callback. + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/0)); + // Steps with large IDs (> int32_t max) should not trigger trace callback. + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), + /*id=*/static_cast(std::numeric_limits::max()) + 1)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), + /*id=*/std::numeric_limits::max())); + // Steps with negative IDs should not trigger trace callback. + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/-1)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(), /*id=*/-100)); + + auto env = NewTestingRuntimeEnv(); + CelExpressionFlatImpl impl( + env, FlatExpression(std::move(path), 0, + env->type_registry.GetComposedTypeProvider(), + cel::RuntimeOptions{})); + + Activation activation; + google::protobuf::Arena arena; + + std::vector traced_ids; + auto eval_status = impl.Trace( + activation, &arena, + [&](int64_t expr_id, const CelValue& value, google::protobuf::Arena* arena) { + traced_ids.push_back(expr_id); + return absl::OkStatus(); + }); + ASSERT_THAT(eval_status, IsOk()); + EXPECT_THAT(traced_ids, ElementsAre(0)); +} + TEST(EvaluatorCoreTest, TraceTest) { Expr expr; cel::expr::SourceInfo source_info; @@ -186,7 +225,7 @@ TEST(EvaluatorCoreTest, TraceTest) { cel::RuntimeOptions options; options.short_circuiting = false; CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - ASSERT_OK(RegisterBuiltinFunctions(builder.GetRegistry())); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); ASSERT_OK_AND_ASSIGN(auto cel_expr, builder.CreateExpression(&expr, &source_info)); @@ -218,7 +257,7 @@ TEST(EvaluatorCoreTest, TraceTest) { callback.Call(expr_id, value, arena); return absl::OkStatus(); }); - ASSERT_OK(eval_status); + ASSERT_THAT(eval_status, IsOk()); } } // namespace google::api::expr::runtime diff --git a/eval/eval/expression_step_base.h b/eval/eval/expression_step_base.h index 5b2f72f8e..565dc5fa5 100644 --- a/eval/eval/expression_step_base.h +++ b/eval/eval/expression_step_base.h @@ -1,11 +1,18 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_BASE_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_BASE_H_ +#include + #include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { -using ExpressionStepBase = ExpressionStep; +class ExpressionStepBase : public ExpressionStepLogic { + public: + ExpressionStepBase() = default; + explicit ExpressionStepBase(int64_t /*expr_id*/, + bool /*comes_from_ast*/ = true) {} +}; } // namespace google::api::expr::runtime diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index eb0f87dc3..ca63392c7 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -157,7 +157,8 @@ class AbstractFunctionStep : public ExpressionStepBase { : ExpressionStepBase(expr_id), name_(name), num_arguments_(num_arguments), - receiver_style_(receiver_style) {} + receiver_style_(receiver_style), + expr_id_(expr_id) {} absl::Status Evaluate(ExecutionFrame* frame) const override; @@ -177,6 +178,7 @@ class AbstractFunctionStep : public ExpressionStepBase { std::string name_; size_t num_arguments_; bool receiver_style_; + int64_t expr_id_; }; inline absl::StatusOr Invoke( @@ -267,7 +269,7 @@ absl::StatusOr AbstractFunctionStep::DoEvaluate( // Overload found and is allowed to consume the arguments. if (matched_function.has_value() && ShouldAcceptOverload(matched_function->descriptor, input_args)) { - return Invoke(*matched_function, id(), input_args, *frame); + return Invoke(*matched_function, expr_id_, input_args, *frame); } return NoOverloadResult(name_, input_args, receiver_style_, *frame); @@ -511,7 +513,7 @@ std::unique_ptr CreateDirectLazyFunctionStep( LazyResolver(std::move(providers), call.function(), call.has_target())); } -absl::StatusOr> CreateFunctionStep( +absl::StatusOr> CreateFunctionStep( const cel::CallExpr& call_expr, int64_t expr_id, std::vector lazy_overloads) { bool receiver_style = call_expr.has_target(); @@ -521,7 +523,7 @@ absl::StatusOr> CreateFunctionStep( std::move(lazy_overloads), expr_id); } -absl::StatusOr> CreateFunctionStep( +absl::StatusOr> CreateFunctionStep( const cel::CallExpr& call_expr, int64_t expr_id, std::vector overloads) { bool receiver_style = call_expr.has_target(); diff --git a/eval/eval/function_step.h b/eval/eval/function_step.h index 9f664dc09..94307609e 100644 --- a/eval/eval/function_step.h +++ b/eval/eval/function_step.h @@ -32,14 +32,14 @@ std::unique_ptr CreateDirectLazyFunctionStep( // Factory method for Call-based execution step where the function will be // resolved at runtime (lazily) from an input Activation. -absl::StatusOr> CreateFunctionStep( +absl::StatusOr> CreateFunctionStep( const cel::CallExpr& call, int64_t expr_id, std::vector lazy_overloads); // Factory method for Call-based execution step where the function has been // statically resolved from a set of eagerly functions configured in the // CelFunctionRegistry. -absl::StatusOr> CreateFunctionStep( +absl::StatusOr> CreateFunctionStep( const cel::CallExpr& call, int64_t expr_id, std::vector overloads); diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index 1bc6b35ee..8adbda76c 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -216,7 +216,8 @@ std::unique_ptr CreateExpressionImpl( const cel::RuntimeOptions& options, std::unique_ptr expr) { ExecutionPath path; - path.push_back(std::make_unique(std::move(expr), -1)); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(std::move(expr)))); auto env = NewTestingRuntimeEnv(); return std::make_unique( @@ -225,17 +226,21 @@ std::unique_ptr CreateExpressionImpl( env->type_registry.GetComposedTypeProvider(), options)); } -absl::StatusOr> MakeTestFunctionStep( +absl::StatusOr MakeTestFunctionStep( const CallExpr& call, const CelFunctionRegistry& registry) { auto argument_matcher = ArgumentMatcher(call); auto lazy_overloads = registry.ModernFindLazyOverloads( call.function(), call.has_target(), argument_matcher); + int id = GetExprId(); if (!lazy_overloads.empty()) { - return CreateFunctionStep(call, GetExprId(), lazy_overloads); + CEL_ASSIGN_OR_RETURN(auto logic, + CreateFunctionStep(call, id, lazy_overloads)); + return ExpressionStep::MakeGenericStep(std::move(logic), id); } auto overloads = registry.FindStaticOverloads( call.function(), call.has_target(), argument_matcher); - return CreateFunctionStep(call, GetExprId(), overloads); + CEL_ASSIGN_OR_RETURN(auto logic, CreateFunctionStep(call, id, overloads)); + return ExpressionStep::MakeGenericStep(std::move(logic), id); } // Test common functions with varying levels of unknown support. @@ -397,7 +402,7 @@ TEST_P(FunctionStepTest, TestNoMatchingOverloadsUnexpectedArgCount) { ASSERT_OK_AND_ASSIGN(auto step2, MakeTestFunctionStep(call1, registry)); ASSERT_OK_AND_ASSIGN( - auto step3, + auto step3_logic, CreateFunctionStep(add_call, -1, registry.FindStaticOverloads( add_call.function(), false, @@ -406,7 +411,7 @@ TEST_P(FunctionStepTest, TestNoMatchingOverloadsUnexpectedArgCount) { path.push_back(std::move(step0)); path.push_back(std::move(step1)); path.push_back(std::move(step2)); - path.push_back(std::move(step3)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step3_logic))); std::unique_ptr impl = GetExpression(std::move(path)); @@ -535,13 +540,11 @@ TEST_P(FunctionStepTest, LazyFunctionOverloadingTest) { lt_call.mutable_args().emplace_back(); lt_call.set_function("_<_"); - ASSERT_OK_AND_ASSIGN( - auto step0, - CreateConstValueStep(cel::interop_internal::CreateIntValue(20), -1)); + auto step0 = ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::interop_internal::CreateIntValue(20))); ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); - ASSERT_OK_AND_ASSIGN( - auto step2, - CreateConstValueStep(cel::interop_internal::CreateDoubleValue(21.9), -1)); + auto step2 = ExpressionStep::MakeGenericStep( + CreateConstValueStep(cel::interop_internal::CreateDoubleValue(21.9))); ASSERT_OK_AND_ASSIGN(auto step3, MakeTestFunctionStep(call2, registry)); ASSERT_OK_AND_ASSIGN(auto step4, MakeTestFunctionStep(lt_call, registry)); @@ -675,8 +678,8 @@ TEST_P(FunctionStepTestUnknowns, PartialUnknownHandlingTest) { IdentExpr ident1; ident1.set_name("param"); CallExpr call1 = SinkFunction::MakeCall(); - - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep("param", GetExprId())); + auto step0 = + ExpressionStep::MakeGenericStep(CreateIdentStep("param"), GetExprId()); ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); path.push_back(std::move(step0)); @@ -987,17 +990,17 @@ TEST(FunctionStepStrictnessTest, IfFunctionStrictAndGivenUnknownSkipsInvocation) { UnknownSet unknown_set; CelFunctionRegistry registry; - ASSERT_OK(registry.Register(std::make_unique( - CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown"))); - ASSERT_OK(registry.Register(std::make_unique( - CelValue::Type::kUnknownSet, /*is_strict=*/true))); + ASSERT_THAT(registry.Register(std::make_unique( + CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown")), + IsOk()); + ASSERT_THAT(registry.Register(std::make_unique( + CelValue::Type::kUnknownSet, /*is_strict=*/true)), + IsOk()); ExecutionPath path; CallExpr call0 = ConstFunction::MakeCall("ConstUnknown"); CallExpr call1 = SinkFunction::MakeCall(); - ASSERT_OK_AND_ASSIGN(std::unique_ptr step0, - MakeTestFunctionStep(call0, registry)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr step1, - MakeTestFunctionStep(call1, registry)); + ASSERT_OK_AND_ASSIGN(auto step0, MakeTestFunctionStep(call0, registry)); + ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); path.push_back(std::move(step0)); path.push_back(std::move(step1)); cel::RuntimeOptions options; @@ -1018,17 +1021,17 @@ TEST(FunctionStepStrictnessTest, TEST(FunctionStepStrictnessTest, IfFunctionNonStrictAndGivenUnknownInvokesIt) { UnknownSet unknown_set; CelFunctionRegistry registry; - ASSERT_OK(registry.Register(std::make_unique( - CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown"))); - ASSERT_OK(registry.Register(std::make_unique( - CelValue::Type::kUnknownSet, /*is_strict=*/false))); + ASSERT_THAT(registry.Register(std::make_unique( + CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown")), + IsOk()); + ASSERT_THAT(registry.Register(std::make_unique( + CelValue::Type::kUnknownSet, /*is_strict=*/false)), + IsOk()); ExecutionPath path; CallExpr call0 = ConstFunction::MakeCall("ConstUnknown"); CallExpr call1 = SinkFunction::MakeCall(); - ASSERT_OK_AND_ASSIGN(std::unique_ptr step0, - MakeTestFunctionStep(call0, registry)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr step1, - MakeTestFunctionStep(call1, registry)); + ASSERT_OK_AND_ASSIGN(auto step0, MakeTestFunctionStep(call0, registry)); + ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); path.push_back(std::move(step0)); path.push_back(std::move(step1)); Expr placeholder_expr; diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index 93c3b1755..9d6edef72 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -29,8 +29,7 @@ using ::cel::runtime_internal::CreateError; class IdentStep : public ExpressionStepBase { public: - IdentStep(absl::string_view name, int64_t expr_id) - : ExpressionStepBase(expr_id), name_(name) {} + explicit IdentStep(absl::string_view name) : name_(name) {} absl::Status Evaluate(ExecutionFrame* frame) const override; @@ -97,8 +96,8 @@ absl::StatusOr LookupSlot( class SlotStep : public ExpressionStepBase { public: - SlotStep(absl::string_view name, size_t slot_index, int64_t expr_id) - : ExpressionStepBase(expr_id), name_(name), slot_index_(slot_index) {} + SlotStep(absl::string_view name, size_t slot_index) + : name_(name), slot_index_(slot_index) {} absl::Status Evaluate(ExecutionFrame* frame) const override { CEL_ASSIGN_OR_RETURN(const ComprehensionSlots::Slot* slot, @@ -163,14 +162,14 @@ std::unique_ptr CreateDirectSlotIdentStep( return std::make_unique(identifier, slot_index, expr_id); } -absl::StatusOr> CreateIdentStep( - const absl::string_view name, int64_t expr_id) { - return std::make_unique(name, expr_id); +std::unique_ptr CreateIdentStep( + const absl::string_view name) { + return std::make_unique(name); } -absl::StatusOr> CreateIdentStepForSlot( - const absl::string_view name, size_t slot_index, int64_t expr_id) { - return std::make_unique(name, slot_index, expr_id); +std::unique_ptr CreateIdentStepForSlot( + const absl::string_view name, size_t slot_index) { + return std::make_unique(name, slot_index); } } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step.h b/eval/eval/ident_step.h index d1bdde388..ad65478bd 100644 --- a/eval/eval/ident_step.h +++ b/eval/eval/ident_step.h @@ -19,12 +19,11 @@ std::unique_ptr CreateDirectSlotIdentStep( absl::string_view identifier, size_t slot_index, int64_t expr_id); // Factory method for Ident - based Execution step -absl::StatusOr> CreateIdentStep( - absl::string_view name, int64_t expr_id); +std::unique_ptr CreateIdentStep(absl::string_view name); // Factory method for identifier that has been assigned to a slot. -absl::StatusOr> CreateIdentStepForSlot( - absl::string_view name, size_t slot_index, int64_t expr_id); +std::unique_ptr CreateIdentStepForSlot( + absl::string_view name, size_t slot_index); } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step_test.cc b/eval/eval/ident_step_test.cc index da5c0dc84..7be2a32dc 100644 --- a/eval/eval/ident_step_test.cc +++ b/eval/eval/ident_step_test.cc @@ -29,6 +29,7 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::Cast; using ::cel::ErrorValue; @@ -46,10 +47,10 @@ using ::testing::HasSubstr; using ::testing::SizeIs; TEST(IdentStepTest, TestIdentStep) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*id=*/-1)); + auto step = CreateIdentStep("name0"); ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( @@ -63,7 +64,7 @@ TEST(IdentStepTest, TestIdentStep) { activation.InsertValue("name0", CelValue::CreateString(&value)); auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); @@ -72,10 +73,10 @@ TEST(IdentStepTest, TestIdentStep) { } TEST(IdentStepTest, TestIdentStepNameNotFound) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*id=*/-1)); + auto step = CreateIdentStep("name0"); ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( @@ -88,17 +89,17 @@ TEST(IdentStepTest, TestIdentStepNameNotFound) { std::string value("test"); auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); ASSERT_TRUE(result.IsError()); } TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*id=*/-1)); + auto step = CreateIdentStep("name0"); ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kDisabled; auto env = NewTestingRuntimeEnv(); @@ -114,7 +115,7 @@ TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { activation.InsertValue("name0", CelValue::CreateString(&value)); auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); @@ -125,16 +126,16 @@ TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { activation.set_missing_attribute_patterns({pattern}); status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); EXPECT_THAT(status0->StringOrDie().value(), Eq("test")); } TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*expr_id=*/1)); + auto step = CreateIdentStep("name0"); ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kDisabled; @@ -153,7 +154,7 @@ TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { activation.InsertValue("name0", CelValue::CreateString(&value)); auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); @@ -164,17 +165,17 @@ TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { activation.set_missing_attribute_patterns({pattern}); status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); EXPECT_EQ(status0->ErrorOrDie()->code(), absl::StatusCode::kInvalidArgument); EXPECT_EQ(status0->ErrorOrDie()->message(), "MissingAttributeError: name0"); } TEST(IdentStepTest, TestIdentStepUnknownAttribute) { - ASSERT_OK_AND_ASSIGN(auto step, CreateIdentStep("name0", /*expr_id=*/1)); + auto step = CreateIdentStep("name0"); ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); // Expression with unknowns enabled. cel::RuntimeOptions options; @@ -196,7 +197,7 @@ TEST(IdentStepTest, TestIdentStepUnknownAttribute) { activation.set_unknown_attribute_patterns(unknown_patterns); auto status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); CelValue result = status0.value(); @@ -207,7 +208,7 @@ TEST(IdentStepTest, TestIdentStepUnknownAttribute) { activation.set_unknown_attribute_patterns(unknown_patterns); status0 = impl.Evaluate(activation, &arena); - ASSERT_OK(status0); + ASSERT_THAT(status0, IsOk()); result = status0.value(); @@ -231,7 +232,7 @@ TEST(DirectIdentStepTest, Basic) { auto step = CreateDirectIdentStep("var1", -1); - ASSERT_OK(step->Evaluate(frame, result, trail)); + ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).NativeValue(), Eq(42)); @@ -256,7 +257,7 @@ TEST(DirectIdentStepTest, UnknownAttribute) { auto step = CreateDirectIdentStep("var1", -1); - ASSERT_OK(step->Evaluate(frame, result, trail)); + ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).ToAttributeSet(), SizeIs(1)); @@ -281,7 +282,7 @@ TEST(DirectIdentStepTest, MissingAttribute) { auto step = CreateDirectIdentStep("var1", -1); - ASSERT_OK(step->Evaluate(frame, result, trail)); + ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).NativeValue(), @@ -303,7 +304,7 @@ TEST(DirectIdentStepTest, NotFound) { auto step = CreateDirectIdentStep("var1", -1); - ASSERT_OK(step->Evaluate(frame, result, trail)); + ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).NativeValue(), diff --git a/eval/eval/jump_step.cc b/eval/eval/jump_step.cc index 03306ac80..ff5d864f2 100644 --- a/eval/eval/jump_step.cc +++ b/eval/eval/jump_step.cc @@ -38,8 +38,8 @@ using ::cel::runtime_internal::CreateNoMatchingOverloadError; class JumpStep : public JumpStepBase { public: // Constructs FunctionStep that uses overloads specified. - JumpStep(absl::optional jump_offset, int64_t expr_id) - : JumpStepBase(jump_offset, expr_id) {} + explicit JumpStep(absl::optional jump_offset) + : JumpStepBase(jump_offset) {} absl::Status Evaluate(ExecutionFrame* frame) const override { return Jump(frame); @@ -49,8 +49,8 @@ class JumpStep : public JumpStepBase { class CondJumpStep : public JumpStepBase { public: CondJumpStep(bool jump_condition, absl::optional jump_offset, - size_t stack_size, int64_t expr_id) - : JumpStepBase(jump_offset, expr_id), + size_t stack_size) + : JumpStepBase(jump_offset), jump_condition_(jump_condition), stack_size_(stack_size) {} @@ -79,8 +79,8 @@ class CondJumpStep : public JumpStepBase { class TernaryCondJumpStep : public JumpStepBase { public: - TernaryCondJumpStep(absl::optional jump_offset, int64_t expr_id) - : JumpStepBase(jump_offset, expr_id) {} + explicit TernaryCondJumpStep(absl::optional jump_offset) + : JumpStepBase(jump_offset) {} absl::Status Evaluate(ExecutionFrame* frame) const override { // Peek the top value @@ -109,8 +109,8 @@ class BoolCheckJumpStep : public JumpStepBase { // - jump to the label if it is unknown value // - jump to the label if it is neither an error nor a boolean, pops it and // pushes "no matching overload" error - BoolCheckJumpStep(absl::optional jump_offset, int64_t expr_id) - : JumpStepBase(jump_offset, expr_id) {} + explicit BoolCheckJumpStep(absl::optional jump_offset) + : JumpStepBase(jump_offset) {} absl::Status Evaluate(ExecutionFrame* frame) const override { // Peek the top value @@ -144,29 +144,28 @@ class BoolCheckJumpStep : public JumpStepBase { // Factory method for Conditional Jump step. std::unique_ptr CreateCondJumpStep( bool jump_condition, absl::optional jump_offset, - size_t expected_stack_size, int64_t expr_id) { + size_t expected_stack_size) { return std::make_unique(jump_condition, jump_offset, - expected_stack_size, expr_id); + expected_stack_size); } // Factory method for Ternary Conditional Jump step. std::unique_ptr CreateTernaryCondJumpStep( - absl::optional jump_offset, int64_t expr_id) { - return std::make_unique(jump_offset, expr_id); + absl::optional jump_offset) { + return std::make_unique(jump_offset); } // Factory method for Jump step. -std::unique_ptr CreateJumpStep(absl::optional jump_offset, - int64_t expr_id) { - return std::make_unique(jump_offset, expr_id); +std::unique_ptr CreateJumpStep(absl::optional jump_offset) { + return std::make_unique(jump_offset); } // Factory method for Conditional Jump step. // Conditional Jump requires a value to sit on the stack. // If this value is an error or unknown, a jump is performed. std::unique_ptr CreateBoolCheckJumpStep( - absl::optional jump_offset, int64_t expr_id) { - return std::make_unique(jump_offset, expr_id); + absl::optional jump_offset) { + return std::make_unique(jump_offset); } } // namespace google::api::expr::runtime diff --git a/eval/eval/jump_step.h b/eval/eval/jump_step.h index d8555ae10..5cef97329 100644 --- a/eval/eval/jump_step.h +++ b/eval/eval/jump_step.h @@ -28,8 +28,8 @@ namespace google::api::expr::runtime { class JumpStepBase : public ExpressionStepBase { public: - JumpStepBase(absl::optional jump_offset, int64_t expr_id) - : ExpressionStepBase(expr_id, false), jump_offset_(jump_offset) {} + explicit JumpStepBase(absl::optional jump_offset = absl::nullopt) + : ExpressionStepBase(), jump_offset_(jump_offset) {} void set_jump_offset(int offset) { jump_offset_ = offset; } @@ -45,8 +45,8 @@ class JumpStepBase : public ExpressionStepBase { }; // Factory method for Jump step. -std::unique_ptr CreateJumpStep(absl::optional jump_offset, - int64_t expr_id); +std::unique_ptr CreateJumpStep( + absl::optional jump_offset = absl::nullopt); // Factory method for Conditional Jump step (used for and/or shortcircuiting). // Conditional Jump requires a boolean value to sit on the stack. @@ -54,20 +54,20 @@ std::unique_ptr CreateJumpStep(absl::optional jump_offset, // The boolean value is left on top of the stack. std::unique_ptr CreateCondJumpStep( bool jump_condition, absl::optional jump_offset, - size_t expected_stack_size, int64_t expr_id); + size_t expected_stack_size); // Factory method for Ternary Conditional Jump step. // Requires a boolean condition value on top of the stack. // If the boolean value is false, a jump is performed to the second branch. // The condition value is popped from the stack before jumping or continuing. std::unique_ptr CreateTernaryCondJumpStep( - absl::optional jump_offset, int64_t expr_id); + absl::optional jump_offset = absl::nullopt); // Factory method for ErrorJump step. // This step performs a Jump when an Error is on the top of the stack. // Value is left on stack if it is a bool or an error. std::unique_ptr CreateBoolCheckJumpStep( - absl::optional jump_offset, int64_t expr_id); + absl::optional jump_offset = absl::nullopt); } // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step.cc b/eval/eval/lazy_init_step.cc index eb9be7796..7d5fa538a 100644 --- a/eval/eval/lazy_init_step.cc +++ b/eval/eval/lazy_init_step.cc @@ -38,8 +38,8 @@ using ::cel::Value; class LazyInitStep final : public ExpressionStepBase { public: - LazyInitStep(size_t slot_index, size_t subexpression_index, int64_t expr_id) - : ExpressionStepBase(expr_id), + LazyInitStep(size_t slot_index, size_t subexpression_index) + : ExpressionStepBase(), slot_index_(slot_index), subexpression_index_(subexpression_index) {} @@ -130,8 +130,8 @@ class AssignSlotAndPopStepStep final : public ExpressionStepBase { class ClearSlotStep : public ExpressionStepBase { public: - explicit ClearSlotStep(size_t slot_index, int64_t expr_id) - : ExpressionStepBase(expr_id), slot_index_(slot_index) {} + explicit ClearSlotStep(size_t slot_index) + : ExpressionStepBase(), slot_index_(slot_index) {} absl::Status Evaluate(ExecutionFrame* frame) const override { frame->comprehension_slots().ClearSlot(slot_index_); @@ -144,8 +144,8 @@ class ClearSlotStep : public ExpressionStepBase { class ClearSlotsStep final : public ExpressionStepBase { public: - explicit ClearSlotsStep(size_t slot_index, size_t slot_count, int64_t expr_id) - : ExpressionStepBase(expr_id), + ClearSlotsStep(size_t slot_index, size_t slot_count) + : ExpressionStepBase(), slot_index_(slot_index), slot_count_(slot_count) {} @@ -210,27 +210,24 @@ std::unique_ptr CreateDirectLazyInitStep( expr_id); } -std::unique_ptr CreateLazyInitStep(size_t slot_index, - size_t subexpression_index, - int64_t expr_id) { - return std::make_unique(slot_index, subexpression_index, - expr_id); +std::unique_ptr CreateLazyInitStep( + size_t slot_index, size_t subexpression_index) { + return std::make_unique(slot_index, subexpression_index); } -std::unique_ptr CreateAssignSlotAndPopStep(size_t slot_index) { +std::unique_ptr CreateAssignSlotAndPopStep( + size_t slot_index) { return std::make_unique(slot_index); } -std::unique_ptr CreateClearSlotStep(size_t slot_index, - int64_t expr_id) { - return std::make_unique(slot_index, expr_id); +std::unique_ptr CreateClearSlotStep(size_t slot_index) { + return std::make_unique(slot_index); } -std::unique_ptr CreateClearSlotsStep(size_t slot_index, - size_t slot_count, - int64_t expr_id) { +std::unique_ptr CreateClearSlotsStep(size_t slot_index, + size_t slot_count) { ABSL_DCHECK_GT(slot_count, 0); - return std::make_unique(slot_index, slot_count, expr_id); + return std::make_unique(slot_index, slot_count); } } // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step.h b/eval/eval/lazy_init_step.h index 714308dfd..e1ef59c93 100644 --- a/eval/eval/lazy_init_step.h +++ b/eval/eval/lazy_init_step.h @@ -65,22 +65,20 @@ std::unique_ptr CreateDirectLazyInitStep( // Creates a step representing accessing a lazily evaluated alias from // a bind or block. -std::unique_ptr CreateLazyInitStep(size_t slot_index, - size_t subexpression_index, - int64_t expr_id); +std::unique_ptr CreateLazyInitStep( + size_t slot_index, size_t subexpression_index); // Helper step to assign a slot value from the top of stack on initialization. -std::unique_ptr CreateAssignSlotAndPopStep(size_t slot_index); +std::unique_ptr CreateAssignSlotAndPopStep( + size_t slot_index); // Helper step to clear a slot. // Slots may be reused in different contexts so need to be cleared after a // context is done. -std::unique_ptr CreateClearSlotStep(size_t slot_index, - int64_t expr_id); +std::unique_ptr CreateClearSlotStep(size_t slot_index); -std::unique_ptr CreateClearSlotsStep(size_t slot_index, - size_t slot_count, - int64_t expr_id); +std::unique_ptr CreateClearSlotsStep(size_t slot_index, + size_t slot_count); } // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step_test.cc b/eval/eval/lazy_init_step_test.cc index b9bef90a1..00490dbe1 100644 --- a/eval/eval/lazy_init_step_test.cc +++ b/eval/eval/lazy_init_step_test.cc @@ -62,11 +62,11 @@ TEST_F(LazyInitStepTest, CreateCheckInitStepDoesInit) { ExecutionPath path; ExecutionPath subpath; - path.push_back(CreateLazyInitStep(/*slot_index=*/0, - /*subexpression_index=*/1, -1)); + path.push_back(ExpressionStep::MakeGenericStep(CreateLazyInitStep( + /*slot_index=*/0, /*subexpression_index=*/1))); - ASSERT_OK_AND_ASSIGN(subpath.emplace_back(), - CreateConstValueStep(cel::IntValue(42), -1, false)); + subpath.push_back( + ExpressionStep::MakeGenericStep(CreateConstValueStep(cel::IntValue(42)))); std::vector expression_table{path, subpath}; @@ -84,10 +84,11 @@ TEST_F(LazyInitStepTest, CreateCheckInitStepSkipInit) { // This is the expected usage, but in this test we are just depending on the // fact that these don't change the stack and fit the program layout // requirements. - path.push_back(CreateLazyInitStep(/*slot_index=*/0, -1, -1)); + path.push_back(ExpressionStep::MakeGenericStep(CreateLazyInitStep( + /*slot_index=*/0, /*subexpression_index=*/-1))); - ASSERT_OK_AND_ASSIGN(subpath.emplace_back(), - CreateConstValueStep(cel::IntValue(42), -1, false)); + subpath.push_back( + ExpressionStep::MakeGenericStep(CreateConstValueStep(cel::IntValue(42)))); std::vector expression_table{path, subpath}; @@ -102,7 +103,8 @@ TEST_F(LazyInitStepTest, CreateCheckInitStepSkipInit) { TEST_F(LazyInitStepTest, CreateAssignSlotAndPopStepBasic) { ExecutionPath path; - path.push_back(CreateAssignSlotAndPopStep(0)); + path.push_back( + ExpressionStep::MakeGenericStep(CreateAssignSlotAndPopStep(0))); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().ClearSlot(0); @@ -122,7 +124,7 @@ TEST_F(LazyInitStepTest, CreateAssignSlotAndPopStepBasic) { TEST_F(LazyInitStepTest, CreateClearSlotStepBasic) { ExecutionPath path; - path.push_back(CreateClearSlotStep(0, -1)); + path.push_back(ExpressionStep::MakeGenericStep(CreateClearSlotStep(0))); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().Set(0, cel::IntValue(42)); @@ -137,7 +139,7 @@ TEST_F(LazyInitStepTest, CreateClearSlotStepBasic) { TEST_F(LazyInitStepTest, CreateClearSlotsStepBasic) { ExecutionPath path; - path.push_back(CreateClearSlotsStep(0, 2, -1)); + path.push_back(ExpressionStep::MakeGenericStep(CreateClearSlotsStep(0, 2))); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().Set(0, cel::IntValue(42)); diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index 84a5a648f..9eaa94f4b 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -191,9 +191,8 @@ absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, class LogicalOpStep : public ExpressionStepBase { public: // Constructs FunctionStep that uses overloads specified. - LogicalOpStep(OpType op_type, size_t count, int64_t expr_id) - : ExpressionStepBase(expr_id), - shortcircuit_(op_type == OpType::kOr), + LogicalOpStep(OpType op_type, size_t count) + : shortcircuit_(op_type == OpType::kOr), op_type_(op_type), count_(count) {} @@ -329,7 +328,7 @@ absl::Status DirectNotStep::Evaluate(ExecutionFrameBase& frame, Value& result, class IterativeNotStep : public ExpressionStepBase { public: - explicit IterativeNotStep(int64_t expr_id) : ExpressionStepBase(expr_id) {} + IterativeNotStep() = default; absl::Status Evaluate(ExecutionFrame* frame) const override; }; @@ -405,8 +404,7 @@ absl::Status DirectNotStrictlyFalseStep::Evaluate( class IterativeNotStrictlyFalseStep : public ExpressionStepBase { public: - explicit IterativeNotStrictlyFalseStep(int64_t expr_id) - : ExpressionStepBase(expr_id) {} + IterativeNotStrictlyFalseStep() = default; absl::Status Evaluate(ExecutionFrame* frame) const override; }; @@ -437,7 +435,6 @@ absl::Status IterativeNotStrictlyFalseStep::Evaluate( } // namespace -// Factory method for "And" Execution step std::unique_ptr CreateDirectAndStep( std::unique_ptr lhs, std::unique_ptr rhs, int64_t expr_id, @@ -446,7 +443,6 @@ std::unique_ptr CreateDirectAndStep( OpType::kAnd, shortcircuiting); } -// Factory method for "Or" Execution step std::unique_ptr CreateDirectOrStep( std::unique_ptr lhs, std::unique_ptr rhs, int64_t expr_id, @@ -456,15 +452,13 @@ std::unique_ptr CreateDirectOrStep( } // Factory method for "And" Execution step -absl::StatusOr> CreateAndStep(size_t num_args, - int64_t expr_id) { - return std::make_unique(OpType::kAnd, num_args, expr_id); +std::unique_ptr CreateAndStep(size_t num_args) { + return std::make_unique(OpType::kAnd, num_args); } // Factory method for "Or" Execution step -absl::StatusOr> CreateOrStep(size_t num_args, - int64_t expr_id) { - return std::make_unique(OpType::kOr, num_args, expr_id); +std::unique_ptr CreateOrStep(size_t num_args) { + return std::make_unique(OpType::kOr, num_args); } // Factory method for recursive logical not "!" Execution step @@ -474,8 +468,8 @@ std::unique_ptr CreateDirectNotStep( } // Factory method for iterative logical not "!" Execution step -std::unique_ptr CreateNotStep(int64_t expr_id) { - return std::make_unique(expr_id); +std::unique_ptr CreateNotStep() { + return std::make_unique(); } // Factory method for recursive logical "@not_strictly_false" Execution step. @@ -486,8 +480,8 @@ std::unique_ptr CreateDirectNotStrictlyFalseStep( } // Factory method for iterative logical "@not_strictly_false" Execution step. -std::unique_ptr CreateNotStrictlyFalseStep(int64_t expr_id) { - return std::make_unique(expr_id); +std::unique_ptr CreateNotStrictlyFalseStep() { + return std::make_unique(); } } // namespace google::api::expr::runtime diff --git a/eval/eval/logic_step.h b/eval/eval/logic_step.h index 4f5be2615..95f58b9db 100644 --- a/eval/eval/logic_step.h +++ b/eval/eval/logic_step.h @@ -23,26 +23,24 @@ std::unique_ptr CreateDirectOrStep( bool shortcircuiting); // Factory method for "And" Execution step -absl::StatusOr> CreateAndStep(size_t num_args, - int64_t expr_id); +std::unique_ptr CreateAndStep(size_t num_args); // Factory method for "Or" Execution step -absl::StatusOr> CreateOrStep(size_t num_args, - int64_t expr_id); +std::unique_ptr CreateOrStep(size_t num_args); // Factory method for recursive logical not "!" Execution step std::unique_ptr CreateDirectNotStep( std::unique_ptr operand, int64_t expr_id); // Factory method for iterative logical not "!" Execution step -std::unique_ptr CreateNotStep(int64_t expr_id); +std::unique_ptr CreateNotStep(); // Factory method for recursive logical "@not_strictly_false" Execution step. std::unique_ptr CreateDirectNotStrictlyFalseStep( std::unique_ptr operand, int64_t expr_id); // Factory method for iterative logical "@not_strictly_false" Execution step. -std::unique_ptr CreateNotStrictlyFalseStep(int64_t expr_id); +std::unique_ptr CreateNotStrictlyFalseStep(); } // namespace google::api::expr::runtime diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index 29725377a..bdb6ceed9 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -67,16 +67,11 @@ class LogicStepTest : public testing::TestWithParam { absl::Status EvaluateLogic(CelValue arg0, CelValue arg1, bool is_or, CelValue* result, bool enable_unknown) { ExecutionPath path; - CEL_ASSIGN_OR_RETURN(auto step, CreateIdentStep("name0", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, CreateIdentStep("name1", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN( - step, (is_or) ? CreateOrStep(/*num_args=*/2, /*expr_id=*/2) - : CreateAndStep(/*num_args=*/2, /*expr_id=*/2)); - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name0"))); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name1"))); + path.push_back(ExpressionStep::MakeGenericStep( + (is_or) ? CreateOrStep(/*num_args=*/2) : CreateAndStep(/*num_args=*/2), + /*expr_id=*/2)); auto dummy_expr = std::make_unique(); cel::RuntimeOptions options; diff --git a/eval/eval/optional_or_step.cc b/eval/eval/optional_or_step.cc index 6e474e404..e98c6ff16 100644 --- a/eval/eval/optional_or_step.cc +++ b/eval/eval/optional_or_step.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include "absl/base/optimization.h" @@ -80,8 +81,8 @@ ErrorValue MakeNoOverloadError(OptionalOrKind kind, google::protobuf::Arena* are // getting the result of optional.value()) class OptionalHasValueJumpStep final : public JumpStepBase { public: - OptionalHasValueJumpStep(int64_t expr_id, OptionalOrKind kind) - : JumpStepBase({}, expr_id), kind_(kind) {} + explicit OptionalHasValueJumpStep(OptionalOrKind kind) + : JumpStepBase(std::nullopt), kind_(kind) {} absl::Status Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(1)) { @@ -111,8 +112,8 @@ class OptionalHasValueJumpStep final : public JumpStepBase { class OptionalOrStep : public ExpressionStepBase { public: - explicit OptionalOrStep(int64_t expr_id, OptionalOrKind kind) - : ExpressionStepBase(expr_id), kind_(kind) {} + explicit OptionalOrStep(OptionalOrKind kind) + : ExpressionStepBase(), kind_(kind) {} absl::Status Evaluate(ExecutionFrame* frame) const override; @@ -275,17 +276,13 @@ absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, } // namespace -std::unique_ptr CreateOptionalHasValueJumpStep(bool or_value, - int64_t expr_id) { +std::unique_ptr CreateOptionalHasValueJumpStep(bool or_value) { return std::make_unique( - expr_id, or_value ? OptionalOrKind::kOrValue : OptionalOrKind::kOrOptional); } -std::unique_ptr CreateOptionalOrStep(bool is_or_value, - int64_t expr_id) { +std::unique_ptr CreateOptionalOrStep(bool is_or_value) { return std::make_unique( - expr_id, is_or_value ? OptionalOrKind::kOrValue : OptionalOrKind::kOrOptional); } diff --git a/eval/eval/optional_or_step.h b/eval/eval/optional_or_step.h index 59977c857..3f149e44b 100644 --- a/eval/eval/optional_or_step.h +++ b/eval/eval/optional_or_step.h @@ -31,13 +31,11 @@ namespace google::api::expr::runtime { // true, performs a jump. If `or_value` is true and we are jumping, // `optional.value` is called and the result replaces the optional at the top of // the stack. -std::unique_ptr CreateOptionalHasValueJumpStep(bool or_value, - int64_t expr_id); +std::unique_ptr CreateOptionalHasValueJumpStep(bool or_value); // Factory method for OptionalOr step, used to implement optional.or and // optional.orValue. -std::unique_ptr CreateOptionalOrStep(bool is_or_value, - int64_t expr_id); +std::unique_ptr CreateOptionalOrStep(bool is_or_value); // Creates a step implementing the short-circuiting optional.or or // optional.orValue step. diff --git a/eval/eval/regex_match_step.cc b/eval/eval/regex_match_step.cc index 2a06de1b8..abc301225 100644 --- a/eval/eval/regex_match_step.cc +++ b/eval/eval/regex_match_step.cc @@ -59,9 +59,8 @@ struct MatchesVisitor final { class RegexMatchStep final : public ExpressionStepBase { public: - RegexMatchStep(int64_t expr_id, std::shared_ptr re2) - : ExpressionStepBase(expr_id, /*comes_from_ast=*/true), - re2_(std::move(re2)) {} + explicit RegexMatchStep(std::shared_ptr re2) + : ExpressionStepBase(), re2_(std::move(re2)) {} absl::Status Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(kNumRegexMatchArguments)) { @@ -127,9 +126,9 @@ std::unique_ptr CreateDirectRegexMatchStep( std::move(re2)); } -absl::StatusOr> CreateRegexMatchStep( - std::shared_ptr re2, int64_t expr_id) { - return std::make_unique(expr_id, std::move(re2)); +absl::StatusOr> CreateRegexMatchStep( + std::shared_ptr re2) { + return std::make_unique(std::move(re2)); } } // namespace google::api::expr::runtime diff --git a/eval/eval/regex_match_step.h b/eval/eval/regex_match_step.h index 1d8a09118..bcbff7bdc 100644 --- a/eval/eval/regex_match_step.h +++ b/eval/eval/regex_match_step.h @@ -29,9 +29,8 @@ std::unique_ptr CreateDirectRegexMatchStep( int64_t expr_id, std::unique_ptr subject, std::shared_ptr re2); -absl::StatusOr> CreateRegexMatchStep( - std::shared_ptr re2, int64_t expr_id); - +absl::StatusOr> CreateRegexMatchStep( + std::shared_ptr re2); } #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_REGEX_MATCH_STEP_H_ diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index ca33a774f..528f12d0f 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -206,10 +206,9 @@ absl::Status PerformOptionalGet(const Value& target, absl::string_view field, // message. class SelectStep : public ExpressionStepBase { public: - SelectStep(absl::string_view field, bool test_field_presence, int64_t expr_id, + SelectStep(absl::string_view field, bool test_field_presence, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) - : ExpressionStepBase(expr_id), - field_(field), + : field_(field), unboxing_option_(enable_wrapper_type_null_unboxing ? ProtoWrapperTypeOptions::kUnsetNull : ProtoWrapperTypeOptions::kUnsetProtoDefault), @@ -461,12 +460,12 @@ bool SupportsCachedFieldDescriptor( class ProtoSelectStep : public SelectStep { public: - ProtoSelectStep(absl::string_view value, int64_t expr_id, + ProtoSelectStep(absl::string_view value, bool enable_wrapper_type_null_unboxing, bool enable_optional_types, const google::protobuf::Descriptor* descriptor, const google::protobuf::FieldDescriptor* field_descriptor) - : SelectStep(value, /*test_field_presence=*/false, expr_id, + : SelectStep(value, /*test_field_presence=*/false, enable_wrapper_type_null_unboxing, enable_optional_types), descriptor_(descriptor), field_descriptor_(field_descriptor) { @@ -541,11 +540,10 @@ absl::Status ProtoSelectStep::EvaluateMessageFieldGet( class ProtoHasStep : public SelectStep { public: - ProtoHasStep(absl::string_view field, int64_t expr_id, - bool enable_wrapper_type_null_unboxing, + ProtoHasStep(absl::string_view field, bool enable_wrapper_type_null_unboxing, bool enable_optional_types, const google::protobuf::Descriptor* descriptor, const google::protobuf::FieldDescriptor* field_descriptor) - : SelectStep(field, /*test_field_presence=*/true, expr_id, + : SelectStep(field, /*test_field_presence=*/true, enable_wrapper_type_null_unboxing, enable_optional_types), descriptor_(descriptor), field_descriptor_(field_descriptor) { @@ -611,25 +609,24 @@ std::unique_ptr CreateDirectSelectStep( } // Factory method for Select - based Execution step -absl::StatusOr> CreateSelectStep( - absl::string_view field, bool test_only, int64_t expr_id, +absl::StatusOr> CreateSelectStep( + absl::string_view field, bool test_only, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { - return std::make_unique(std::move(field), test_only, expr_id, + return std::make_unique(field, test_only, enable_wrapper_type_null_unboxing, enable_optional_types); } // Factory method for Select - based Execution step -absl::StatusOr> CreateTypedSelectStep( +absl::StatusOr> CreateTypedSelectStep( absl::string_view field, cel::StructType resolved_operand_type, - cel::StructTypeField resolved_field, bool test_only, int64_t expr_id, + cel::StructTypeField resolved_field, bool test_only, bool enable_wrapper_type_null_unboxing, bool enable_optional_types) { if (!resolved_operand_type.IsMessage()) { // The specialization only supports messages. Fallback to the generic // implementation for other types. // TODO(uncreated-issue/89): support optional select and chaining. - return CreateSelectStep(std::move(field), test_only, expr_id, - enable_wrapper_type_null_unboxing, + return CreateSelectStep(field, test_only, enable_wrapper_type_null_unboxing, enable_optional_types); } const google::protobuf::Descriptor* descriptor = @@ -648,20 +645,19 @@ absl::StatusOr> CreateTypedSelectStep( // crash. // // Fallback to the generic implementation. - return CreateSelectStep(std::move(field), test_only, expr_id, - enable_wrapper_type_null_unboxing, + return CreateSelectStep(field, test_only, enable_wrapper_type_null_unboxing, enable_optional_types); } if (test_only) { return std::make_unique( - std::move(field), expr_id, enable_wrapper_type_null_unboxing, - enable_optional_types, descriptor, field_descriptor); + field, enable_wrapper_type_null_unboxing, enable_optional_types, + descriptor, field_descriptor); } return std::make_unique( - std::move(field), expr_id, enable_wrapper_type_null_unboxing, - enable_optional_types, descriptor, field_descriptor); + field, enable_wrapper_type_null_unboxing, enable_optional_types, + descriptor, field_descriptor); } } // namespace google::api::expr::runtime diff --git a/eval/eval/select_step.h b/eval/eval/select_step.h index 17a93d19f..528e90627 100644 --- a/eval/eval/select_step.h +++ b/eval/eval/select_step.h @@ -19,13 +19,13 @@ std::unique_ptr CreateDirectSelectStep( bool enable_optional_types = false); // Factory method for Select stack machine based Execution step -absl::StatusOr> CreateSelectStep( - absl::string_view field, bool test_only, int64_t expr_id, - bool enable_wrapper_type_null_unboxing, bool enable_optional_ytpes = false); +absl::StatusOr> CreateSelectStep( + absl::string_view field, bool test_only, + bool enable_wrapper_type_null_unboxing, bool enable_optional_types = false); -absl::StatusOr> CreateTypedSelectStep( +absl::StatusOr> CreateTypedSelectStep( absl::string_view field, cel::StructType resolved_operand_type, - cel::StructTypeField resolved_field, bool test_only, int64_t expr_id, + cel::StructTypeField resolved_field, bool test_only, bool enable_wrapper_type_null_unboxing, bool enable_optional_types); } // namespace google::api::expr::runtime diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index b9925fb52..75b987463 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -104,14 +104,16 @@ class SelectStepTest : public testing::Test { auto& ident = expr0.mutable_ident_expr(); ident.set_name("target"); - CEL_ASSIGN_OR_RETURN(auto step0, CreateIdentStep(ident.name(), expr0.id())); + auto step0 = CreateIdentStep(ident.name()); CEL_ASSIGN_OR_RETURN( auto step1, - CreateSelectStep(select.field(), select.test_only(), expr.id(), + CreateSelectStep(select.field(), select.test_only(), options.enable_wrapper_type_null_unboxing)); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step1), expr.id())); cel::RuntimeOptions runtime_options; if (options.enable_unknowns) { @@ -286,21 +288,21 @@ TEST_F(SelectStepTest, MapPresenseIsErrorTest) { Expr& expr0 = select_map.mutable_operand(); auto& ident = expr0.mutable_ident_expr(); ident.set_name("target"); - - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); + auto step0 = CreateIdentStep(ident.name()); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select_map.field(), select_map.test_only(), expr1.id(), + CreateSelectStep(select_map.field(), select_map.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); ASSERT_OK_AND_ASSIGN( auto step2, - CreateSelectStep(select.field(), select.test_only(), select_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); ExecutionPath path; - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); - path.push_back(std::move(step2)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1), expr1.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step2), select_expr.id())); CelExpressionFlatImpl cel_expr( env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, env_->type_registry.GetComposedTypeProvider(), @@ -747,14 +749,15 @@ TEST_P(SelectStepConformanceTest, CelErrorAsArgument) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); + auto step0 = CreateIdentStep(ident.name()); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select.field(), select.test_only(), dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); CelError error = absl::CancelledError(); @@ -788,14 +791,15 @@ TEST_F(SelectStepTest, DisableMissingAttributeOK) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); + auto step0 = CreateIdentStep(ident.name()); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select.field(), select.test_only(), dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); CelExpressionFlatImpl cel_expr( env_, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, @@ -830,14 +834,15 @@ TEST_F(SelectStepTest, UnrecoverableUnknownValueProducesError) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep(ident.name(), expr0.id())); + auto step0 = CreateIdentStep(ident.name()); ASSERT_OK_AND_ASSIGN( auto step1, - CreateSelectStep(select.field(), select.test_only(), dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back( + ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); cel::RuntimeOptions options; options.enable_missing_attribute_errors = true; @@ -878,16 +883,16 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - auto step0_status = CreateIdentStep(ident.name(), expr0.id()); + auto step0 = CreateIdentStep(ident.name()); auto step1_status = - CreateSelectStep(select.field(), select.test_only(), dummy_expr.id(), + CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false); - ASSERT_THAT(step0_status, IsOk()); ASSERT_THAT(step1_status, IsOk()); - path.push_back(*std::move(step0_status)); - path.push_back(*std::move(step1_status)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back(ExpressionStep::MakeGenericStep(std::move(*step1_status), + dummy_expr.id())); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; @@ -972,7 +977,7 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { } TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep("message", -1)); + auto step0 = CreateIdentStep("message"); cel::StructType resolved_operand_type( (cel::MessageType(TestAllTypes::descriptor()))); @@ -984,13 +989,13 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { ASSERT_OK_AND_ASSIGN( auto step1, CreateTypedSelectStep( "single_int64", resolved_operand_type, resolved_field, - /*test_only=*/false, -1, + /*test_only=*/false, /*enable_wrapper_type_null_unboxing=*/false, /*enable_optional_types=*/false)); ExecutionPath path; - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0))); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); cel::RuntimeOptions options; if (GetParam()) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; @@ -1012,7 +1017,7 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { } TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep("message", -1)); + auto step0 = CreateIdentStep("message"); cel::StructType resolved_operand_type( (cel::MessageType(TestAllTypes::descriptor()))); @@ -1024,13 +1029,13 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { ASSERT_OK_AND_ASSIGN( auto step1, CreateTypedSelectStep( "single_int64", resolved_operand_type, resolved_field, - /*test_only=*/false, -1, + /*test_only=*/false, /*enable_wrapper_type_null_unboxing=*/false, /*enable_optional_types=*/false)); ExecutionPath path; - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0))); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); cel::RuntimeOptions options; if (GetParam()) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; @@ -1049,7 +1054,7 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { } TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { - ASSERT_OK_AND_ASSIGN(auto step0, CreateIdentStep("message", -1)); + auto step0 = CreateIdentStep("message"); cel::StructType resolved_operand_type( (cel::MessageType(TestAllTypes::descriptor()))); @@ -1061,13 +1066,13 @@ TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { ASSERT_OK_AND_ASSIGN( auto step1, CreateTypedSelectStep( "single_int64", resolved_operand_type, resolved_field, - /*test_only=*/false, -1, + /*test_only=*/false, /*enable_wrapper_type_null_unboxing=*/false, /*enable_optional_types=*/false)); ExecutionPath path; - path.push_back(std::move(step0)); - path.push_back(std::move(step1)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step0))); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; CelExpressionFlatImpl cel_expr( diff --git a/eval/eval/shadowable_value_step.cc b/eval/eval/shadowable_value_step.cc index 0839d28cc..0b75f7cc2 100644 --- a/eval/eval/shadowable_value_step.cc +++ b/eval/eval/shadowable_value_step.cc @@ -24,8 +24,8 @@ using ::cel::Value; class ShadowableValueStep : public ExpressionStepBase { public: - ShadowableValueStep(std::string identifier, cel::Value value, int64_t expr_id) - : ExpressionStepBase(expr_id), + ShadowableValueStep(std::string identifier, cel::Value value) + : ExpressionStepBase(), identifier_(std::move(identifier)), value_(std::move(value)) {} @@ -83,10 +83,10 @@ absl::Status DirectShadowableValueStep::Evaluate( } // namespace -absl::StatusOr> CreateShadowableValueStep( - absl::string_view name, cel::Value value, int64_t expr_id) { +absl::StatusOr> CreateShadowableValueStep( + absl::string_view name, cel::Value value) { return std::make_unique(std::string(name), - std::move(value), expr_id); + std::move(value)); } std::unique_ptr CreateDirectShadowableValueStep( diff --git a/eval/eval/shadowable_value_step.h b/eval/eval/shadowable_value_step.h index 9c386f02d..fae90353b 100644 --- a/eval/eval/shadowable_value_step.h +++ b/eval/eval/shadowable_value_step.h @@ -15,8 +15,8 @@ namespace google::api::expr::runtime { // Create an identifier resolution step with a default value that may be // shadowed by an identifier of the same name within the runtime-provided // Activation. -absl::StatusOr> CreateShadowableValueStep( - absl::string_view name, cel::Value value, int64_t expr_id); +absl::StatusOr> CreateShadowableValueStep( + absl::string_view name, cel::Value value); std::unique_ptr CreateDirectShadowableValueStep( absl::string_view name, cel::Value value, int64_t expr_id); diff --git a/eval/eval/shadowable_value_step_test.cc b/eval/eval/shadowable_value_step_test.cc index 4a7cabea1..83ffd5857 100644 --- a/eval/eval/shadowable_value_step_test.cc +++ b/eval/eval/shadowable_value_step_test.cc @@ -34,11 +34,10 @@ absl::StatusOr RunShadowableExpression( const absl_nonnull std::shared_ptr& env, std::string identifier, cel::Value value, const Activation& activation, Arena* arena) { - CEL_ASSIGN_OR_RETURN( - auto step, - CreateShadowableValueStep(std::move(identifier), std::move(value), 1)); + CEL_ASSIGN_OR_RETURN(auto step, + CreateShadowableValueStep(identifier, std::move(value))); ExecutionPath path; - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(std::move(step), 1)); CelExpressionFlatImpl impl( env, FlatExpression(std::move(path), /*comprehension_slot_count=*/0, @@ -55,11 +54,10 @@ TEST(ShadowableValueStepTest, TestEvaluateNoShadowing) { Arena arena; auto type_value = CreateTypeValueFromView(&arena, type_name); - auto status = - RunShadowableExpression(env, type_name, type_value, activation, &arena); - ASSERT_OK(status); + ASSERT_OK_AND_ASSIGN( + auto value, + RunShadowableExpression(env, type_name, type_value, activation, &arena)); - auto value = status.value(); ASSERT_TRUE(value.IsCelType()); EXPECT_THAT(value.CelTypeOrDie().value(), Eq(type_name)); } @@ -74,11 +72,10 @@ TEST(ShadowableValueStepTest, TestEvaluateShadowedIdentifier) { Arena arena; auto type_value = CreateTypeValueFromView(&arena, type_name); - auto status = - RunShadowableExpression(env, type_name, type_value, activation, &arena); - ASSERT_OK(status); + ASSERT_OK_AND_ASSIGN( + auto value, + RunShadowableExpression(env, type_name, type_value, activation, &arena)); - auto value = status.value(); ASSERT_TRUE(value.IsInt64()); EXPECT_THAT(value.Int64OrDie(), Eq(1024L)); } diff --git a/eval/eval/ternary_step.cc b/eval/eval/ternary_step.cc index ea543ab70..97b064d22 100644 --- a/eval/eval/ternary_step.cc +++ b/eval/eval/ternary_step.cc @@ -126,7 +126,7 @@ class ShortcircuitingDirectTernaryStep : public DirectExpressionStep { class TernaryStep : public ExpressionStepBase { public: // Constructs FunctionStep that uses overloads specified. - explicit TernaryStep(int64_t expr_id) : ExpressionStepBase(expr_id) {} + TernaryStep() : ExpressionStepBase() {} absl::Status Evaluate(ExecutionFrame* frame) const override; }; @@ -189,9 +189,8 @@ std::unique_ptr CreateDirectTernaryStep( std::move(condition), std::move(left), std::move(right), expr_id); } -absl::StatusOr> CreateTernaryStep( - int64_t expr_id) { - return std::make_unique(expr_id); +std::unique_ptr CreateTernaryStep() { + return std::make_unique(); } } // namespace google::api::expr::runtime diff --git a/eval/eval/ternary_step.h b/eval/eval/ternary_step.h index 2b51e95ea..82491f589 100644 --- a/eval/eval/ternary_step.h +++ b/eval/eval/ternary_step.h @@ -18,8 +18,7 @@ std::unique_ptr CreateDirectTernaryStep( bool shortcircuiting = true); // Factory method for ternary (_?_:_) execution step -absl::StatusOr> CreateTernaryStep( - int64_t expr_id); +std::unique_ptr CreateTernaryStep(); } // namespace google::api::expr::runtime diff --git a/eval/eval/ternary_step_test.cc b/eval/eval/ternary_step_test.cc index 2c400b3e1..9b17b5356 100644 --- a/eval/eval/ternary_step_test.cc +++ b/eval/eval/ternary_step_test.cc @@ -38,6 +38,7 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::BoolValue; using ::cel::Cast; @@ -64,17 +65,10 @@ class LogicStepTest : public testing::TestWithParam { CelValue* result, bool enable_unknown) { ExecutionPath path; - CEL_ASSIGN_OR_RETURN(auto step, CreateIdentStep("name0", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, CreateIdentStep("name1", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, CreateIdentStep("name2", /*expr_id=*/-1)); - path.push_back(std::move(step)); - - CEL_ASSIGN_OR_RETURN(step, CreateTernaryStep(4)); - path.push_back(std::move(step)); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name0"))); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name1"))); + path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name2"))); + path.push_back(ExpressionStep::MakeGenericStep(CreateTernaryStep(), 4)); cel::RuntimeOptions options; if (enable_unknown) { @@ -109,14 +103,14 @@ TEST_P(LogicStepTest, TestBoolCond) { absl::Status status = EvaluateLogic(CelValue::CreateBool(true), CelValue::CreateBool(true), CelValue::CreateBool(false), &result, GetParam()); - ASSERT_OK(status); + ASSERT_THAT(status, IsOk()); ASSERT_TRUE(result.IsBool()); ASSERT_TRUE(result.BoolOrDie()); status = EvaluateLogic(CelValue::CreateBool(false), CelValue::CreateBool(true), CelValue::CreateBool(false), &result, GetParam()); - ASSERT_OK(status); + ASSERT_THAT(status, IsOk()); ASSERT_TRUE(result.IsBool()); ASSERT_FALSE(result.BoolOrDie()); } @@ -125,16 +119,19 @@ TEST_P(LogicStepTest, TestErrorHandling) { CelValue result; CelError error = absl::CancelledError(); CelValue error_value = CelValue::CreateError(&error); - ASSERT_OK(EvaluateLogic(error_value, CelValue::CreateBool(true), - CelValue::CreateBool(false), &result, GetParam())); + ASSERT_THAT(EvaluateLogic(error_value, CelValue::CreateBool(true), + CelValue::CreateBool(false), &result, GetParam()), + IsOk()); ASSERT_TRUE(result.IsError()); - ASSERT_OK(EvaluateLogic(CelValue::CreateBool(true), error_value, - CelValue::CreateBool(false), &result, GetParam())); + ASSERT_THAT(EvaluateLogic(CelValue::CreateBool(true), error_value, + CelValue::CreateBool(false), &result, GetParam()), + IsOk()); ASSERT_TRUE(result.IsError()); - ASSERT_OK(EvaluateLogic(CelValue::CreateBool(false), error_value, - CelValue::CreateBool(false), &result, GetParam())); + ASSERT_THAT(EvaluateLogic(CelValue::CreateBool(false), error_value, + CelValue::CreateBool(false), &result, GetParam()), + IsOk()); ASSERT_TRUE(result.IsBool()); ASSERT_FALSE(result.BoolOrDie()); } @@ -145,25 +142,30 @@ TEST_F(LogicStepTest, TestUnknownHandling) { CelError cel_error = absl::CancelledError(); CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); CelValue error_value = CelValue::CreateError(&cel_error); - ASSERT_OK(EvaluateLogic(unknown_value, CelValue::CreateBool(true), - CelValue::CreateBool(false), &result, true)); + ASSERT_THAT(EvaluateLogic(unknown_value, CelValue::CreateBool(true), + CelValue::CreateBool(false), &result, true), + IsOk()); ASSERT_TRUE(result.IsUnknownSet()); - ASSERT_OK(EvaluateLogic(CelValue::CreateBool(true), unknown_value, - CelValue::CreateBool(false), &result, true)); + ASSERT_THAT(EvaluateLogic(CelValue::CreateBool(true), unknown_value, + CelValue::CreateBool(false), &result, true), + IsOk()); ASSERT_TRUE(result.IsUnknownSet()); - ASSERT_OK(EvaluateLogic(CelValue::CreateBool(false), unknown_value, - CelValue::CreateBool(false), &result, true)); + ASSERT_THAT(EvaluateLogic(CelValue::CreateBool(false), unknown_value, + CelValue::CreateBool(false), &result, true), + IsOk()); ASSERT_TRUE(result.IsBool()); ASSERT_FALSE(result.BoolOrDie()); - ASSERT_OK(EvaluateLogic(error_value, unknown_value, - CelValue::CreateBool(false), &result, true)); + ASSERT_THAT(EvaluateLogic(error_value, unknown_value, + CelValue::CreateBool(false), &result, true), + IsOk()); ASSERT_TRUE(result.IsError()); - ASSERT_OK(EvaluateLogic(unknown_value, error_value, - CelValue::CreateBool(false), &result, true)); + ASSERT_THAT(EvaluateLogic(unknown_value, error_value, + CelValue::CreateBool(false), &result, true), + IsOk()); ASSERT_TRUE(result.IsUnknownSet()); Expr expr0; @@ -184,9 +186,10 @@ TEST_F(LogicStepTest, TestUnknownHandling) { EXPECT_THAT(unknown_attr_set0.size(), Eq(1)); EXPECT_THAT(unknown_attr_set1.size(), Eq(1)); - ASSERT_OK(EvaluateLogic(CelValue::CreateUnknownSet(&unknown_set0), - CelValue::CreateUnknownSet(&unknown_set1), - CelValue::CreateBool(false), &result, true)); + ASSERT_THAT(EvaluateLogic(CelValue::CreateUnknownSet(&unknown_set0), + CelValue::CreateUnknownSet(&unknown_set1), + CelValue::CreateBool(false), &result, true), + IsOk()); ASSERT_TRUE(result.IsUnknownSet()); const auto& attrs = result.UnknownSetOrDie()->unknown_attributes(); ASSERT_THAT(attrs, testing::SizeIs(1)); @@ -222,7 +225,7 @@ TEST_P(TernaryStepDirectTest, ReturnLhs) { cel::Value result; AttributeTrail attr_unused; - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); + ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_EQ(Cast(result).NativeValue(), 1); @@ -243,7 +246,7 @@ TEST_P(TernaryStepDirectTest, ReturnRhs) { cel::Value result; AttributeTrail attr_unused; - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); + ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_EQ(Cast(result).NativeValue(), 2); @@ -266,7 +269,7 @@ TEST_P(TernaryStepDirectTest, ForwardError) { cel::Value result; AttributeTrail attr_unused; - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); + ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).NativeValue(), @@ -294,7 +297,7 @@ TEST_P(TernaryStepDirectTest, ForwardUnknown) { cel::Value result; AttributeTrail attr_unused; - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); + ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).ToAttributeSet(), ElementsAre(Truly([](const cel::Attribute& attr) { @@ -317,7 +320,7 @@ TEST_P(TernaryStepDirectTest, UnexpectedCondtionKind) { cel::Value result; AttributeTrail attr_unused; - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); + ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).NativeValue(), @@ -358,7 +361,7 @@ TEST_P(TernaryStepDirectTest, Shortcircuiting) { cel::Value result; AttributeTrail attr_unused; - ASSERT_OK(step->Evaluate(frame, result, attr_unused)); + ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); ASSERT_TRUE(InstanceOf(result)); EXPECT_THAT(Cast(result).NativeValue(), Eq(1)); diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 2524e57d1..60d414de3 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -84,6 +84,7 @@ using ::google::api::expr::runtime::CelValue; using ::google::api::expr::runtime::DirectExpressionStep; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionFrameBase; +using ::google::api::expr::runtime::ExpressionStep; using ::google::api::expr::runtime::ExpressionStepBase; using ::google::api::expr::runtime::GetGenericProtoTypeInfoInstance; using ::google::api::expr::runtime::PlannerContext; @@ -972,8 +973,9 @@ absl::Status SelectOptimizer::OnPostVisit(PlannerContext& context, CEL_ASSIGN_OR_RETURN(auto operand_subplan, context.ExtractSubplan(operand)); absl::c_move(operand_subplan, std::back_inserter(path)); - path.push_back( - std::make_unique(node.id(), std::move(impl))); + path.push_back(ExpressionStep::MakeGenericStep( + std::make_unique(node.id(), std::move(impl)), + node.id())); return context.ReplaceSubplan(node, std::move(path)); } diff --git a/runtime/internal/runtime_impl.cc b/runtime/internal/runtime_impl.cc index 92d097b2c..3e8eef59a 100644 --- a/runtime/internal/runtime_impl.cc +++ b/runtime/internal/runtime_impl.cc @@ -139,11 +139,15 @@ RuntimeImpl::CreateTraceableProgram( !flat_expr.subexpressions().empty() && // mainline expression is exactly one recursive step. flat_expr.subexpressions().front().size() == 1 && - flat_expr.subexpressions().front().front()->GetNativeTypeId() == - NativeTypeId::For()) { + flat_expr.subexpressions().front().front().IsGenericStep() && + flat_expr.subexpressions() + .front() + .front() + .GetGenericStep() + ->GetNativeTypeId() == NativeTypeId::For()) { const DirectExpressionStep* root = internal::down_cast( - flat_expr.subexpressions().front().front().get()) + flat_expr.subexpressions().front().front().GetGenericStep()) ->wrapped(); return std::make_unique(environment_, std::move(flat_expr), root); From 80a9b8a3bc5f8f137059352322ac379b9ece914f Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 28 Sep 2026 17:40:08 -0700 Subject: [PATCH 243/350] Cleanup `AttributeQualifier` and `AttributeQualifierPattern` and introduce `AttributeQualifierView` PiperOrigin-RevId: 989943852 --- base/BUILD | 16 +- base/attribute.cc | 486 ++++++++++----- base/attribute.h | 789 +++++++++++++++++++++--- base/attribute_test.cc | 80 +++ eval/public/cel_attribute_test.cc | 8 +- extensions/protobuf/internal/qualify.cc | 6 +- 6 files changed, 1152 insertions(+), 233 deletions(-) create mode 100644 base/attribute_test.cc diff --git a/base/BUILD b/base/BUILD index 600889182..739fc96de 100644 --- a/base/BUILD +++ b/base/BUILD @@ -38,12 +38,24 @@ cc_library( "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/functional:overload", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:optional_ref", "@com_google_absl//absl/types:span", - "@com_google_absl//absl/types:variant", + ], +) + +cc_test( + name = "attribute_test", + srcs = ["attribute_test.cc"], + deps = [ + ":attributes", + "//internal:testing", + "@com_google_absl//absl/strings:string_view", ], ) diff --git a/base/attribute.cc b/base/attribute.cc index f750a1850..97fa75fa7 100644 --- a/base/attribute.cc +++ b/base/attribute.cc @@ -17,13 +17,17 @@ #include #include #include +#include +#include #include "absl/base/macros.h" #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/meta/type_traits.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" -#include "absl/types/variant.h" +#include "absl/strings/string_view.h" #include "base/kind.h" #include "internal/status_macros.h" @@ -36,14 +40,10 @@ class AttributeStringPrinter { public: // String representation for the given qualifier is appended to output. // output must be non-null. - explicit AttributeStringPrinter(std::string* output, Kind type) - : output_(*output), type_(type) {} + explicit AttributeStringPrinter(std::string* output) : output_(*output) {} - absl::Status operator()(const Kind& ignored) const { - // Attributes are represented as a variant, with illegal attribute - // qualifiers represented with their type as the first alternative. - return absl::InvalidArgumentError( - absl::StrCat("Unsupported attribute qualifier ", KindToString(type_))); + absl::Status operator()(std::monostate) const { + return absl::InvalidArgumentError("bad attribute qualifier"); } absl::Status operator()(int64_t index) { @@ -68,22 +68,19 @@ class AttributeStringPrinter { private: std::string& output_; - Kind type_; }; // Visitor for appending string representation for different qualifier kinds. class AttributeQualifierStringPrinter { public: // String representation for the given qualifier is appended to output. - explicit AttributeQualifierStringPrinter(std::string* absl_nonnull output, - Kind type) - : output_(*output), type_(type) {} + explicit AttributeQualifierStringPrinter(std::string* absl_nonnull output) + : output_(*output) {} - absl::Status operator()(const Kind& ignored) const { + absl::Status operator()(std::monostate) const { // Attributes are represented as a variant, with illegal attribute // qualifiers represented with their type as the first alternative. - return absl::InvalidArgumentError( - absl::StrCat("Unsupported attribute qualifier ", KindToString(type_))); + return absl::InvalidArgumentError("bad attribute qualifier"); } absl::Status operator()(int64_t index) { @@ -108,140 +105,24 @@ class AttributeQualifierStringPrinter { private: std::string& output_; - Kind type_; }; struct AttributeQualifierTypeVisitor final { - Kind operator()(const Kind& type) const { return type; } - - Kind operator()(int64_t ignored) const { - static_cast(ignored); - return Kind::kInt64; - } - - Kind operator()(uint64_t ignored) const { - static_cast(ignored); - return Kind::kUint64; - } - - Kind operator()(const std::string& ignored) const { - static_cast(ignored); - return Kind::kString; - } - - Kind operator()(bool ignored) const { - static_cast(ignored); - return Kind::kBool; - } -}; - -struct AttributeQualifierTypeComparator final { - const Kind lhs; - - bool operator()(const Kind& rhs) const { - return static_cast(lhs) < static_cast(rhs); - } - - bool operator()(int64_t) const { return false; } - - bool operator()(uint64_t other) const { return false; } - - bool operator()(const std::string&) const { return false; } - - bool operator()(bool other) const { return false; } -}; - -struct AttributeQualifierIntComparator final { - const int64_t lhs; - - bool operator()(const Kind&) const { return true; } - - bool operator()(int64_t rhs) const { return lhs < rhs; } + Kind operator()(std::monostate) const { return Kind::kNull; } - bool operator()(uint64_t) const { return true; } + Kind operator()(int64_t) const { return Kind::kInt64; } - bool operator()(const std::string&) const { return true; } + Kind operator()(uint64_t) const { return Kind::kUint64; } - bool operator()(bool) const { return false; } -}; - -struct AttributeQualifierUintComparator final { - const uint64_t lhs; - - bool operator()(const Kind&) const { return true; } - - bool operator()(int64_t) const { return false; } - - bool operator()(uint64_t rhs) const { return lhs < rhs; } - - bool operator()(const std::string&) const { return true; } - - bool operator()(bool) const { return false; } -}; - -struct AttributeQualifierStringComparator final { - const std::string& lhs; - - bool operator()(const Kind&) const { return true; } - - bool operator()(int64_t) const { return false; } - - bool operator()(uint64_t) const { return false; } + Kind operator()(const std::string&) const { return Kind::kString; } - bool operator()(const std::string& rhs) const { return lhs < rhs; } - - bool operator()(bool) const { return false; } -}; - -struct AttributeQualifierBoolComparator final { - const bool lhs; - - bool operator()(const Kind&) const { return true; } - - bool operator()(int64_t) const { return true; } - - bool operator()(uint64_t) const { return true; } - - bool operator()(const std::string&) const { return true; } - - bool operator()(bool rhs) const { return lhs < rhs; } + Kind operator()(bool) const { return Kind::kBool; } }; } // namespace -struct AttributeQualifier::ComparatorVisitor final { - const AttributeQualifier::Variant& rhs; - - bool operator()(const Kind& lhs) const { - return absl::visit(AttributeQualifierTypeComparator{lhs}, rhs); - } - - bool operator()(int64_t lhs) const { - return absl::visit(AttributeQualifierIntComparator{lhs}, rhs); - } - - bool operator()(uint64_t lhs) const { - return absl::visit(AttributeQualifierUintComparator{lhs}, rhs); - } - - bool operator()(const std::string& lhs) const { - return absl::visit(AttributeQualifierStringComparator{lhs}, rhs); - } - - bool operator()(bool lhs) const { - return absl::visit(AttributeQualifierBoolComparator{lhs}, rhs); - } -}; - Kind AttributeQualifier::kind() const { - return absl::visit(AttributeQualifierTypeVisitor{}, value_); -} - -bool AttributeQualifier::operator<(const AttributeQualifier& other) const { - // The order is not publicly documented because it is subject to change. - // Currently we sort in the following order, with each type being sorted - // against itself: bool, int, uint, string, type. - return absl::visit(ComparatorVisitor{other.value_}, value_); + return std::visit(AttributeQualifierTypeVisitor{}, value_); } bool Attribute::operator==(const Attribute& other) const { @@ -296,7 +177,7 @@ bool Attribute::operator<(const Attribute& other) const { return false; } -const absl::StatusOr Attribute::AsString() const { +absl::StatusOr Attribute::AsString() const { if (variable_name().empty()) { return absl::InvalidArgumentError( "Only ident rooted attributes are supported."); @@ -305,26 +186,335 @@ const absl::StatusOr Attribute::AsString() const { std::string result = std::string(variable_name()); for (const auto& qualifier : qualifier_path()) { - CEL_RETURN_IF_ERROR(absl::visit( - AttributeStringPrinter(&result, qualifier.kind()), qualifier.value_)); + CEL_RETURN_IF_ERROR(std::visit(AttributeStringPrinter(&result), + common_internal::AsVariant(qualifier))); } return result; } +std::string AttributeQualifier::ToString() const { + std::string result; + absl::Status status = + std::visit(AttributeQualifierStringPrinter{&result}, value_); + ABSL_DCHECK_OK(status) << "bad attribute qualifier"; + status.IgnoreError(); + return result; +} + bool AttributeQualifier::IsMatch(const AttributeQualifier& other) const { - if (absl::holds_alternative(value_) || - absl::holds_alternative(other.value_)) { + ABSL_DCHECK(*this) << "bad attribute qualifier"; + ABSL_DCHECK(other) << "bad attribute qualifier"; + if (std::holds_alternative(value_) || + std::holds_alternative(other.value_)) { return false; } - return value_ == other.value_; + return *this == other; } -absl::StatusOr AttributeQualifier::AsString() const { - std::string result; - CEL_RETURN_IF_ERROR( - absl::visit(AttributeQualifierStringPrinter(&result, kind()), value_)); - return result; +bool AttributeQualifier::IsMatch(absl::string_view other_key) const { + ABSL_DCHECK(*this) << "bad attribute qualifier"; + if (auto string = AsString(); string.has_value()) { + return *string == other_key; + } + return false; +} + +bool AttributeQualifierPattern::IsMatch( + const AttributeQualifier& qualifier) const { + ABSL_DCHECK(*this) << "bad attribute qualifier pattern"; + ABSL_DCHECK(qualifier) << "bad attribute qualifier"; + if (!*this || !qualifier) { + return false; + } + if (IsWildcard()) { + return true; + } + return *this == qualifier; +} + +bool AttributeQualifierPattern::IsMatch(absl::string_view other_key) const { + ABSL_DCHECK(*this) << "bad attribute qualifier pattern"; + if (IsWildcard()) { + return true; + } + if (auto string = AsString(); string.has_value()) { + return *string == other_key; + } + return false; +} + +namespace { + +struct AttributeQualifierEqualTo { + bool operator()(const std::monostate&, const std::monostate&) const { + return false; + } + + template + std::enable_if_t, std::monostate>, + bool> + operator()(const std::monostate&, const T&) const { + return false; + } + + template + std::enable_if_t, std::monostate>, + bool> + operator()(const T&, const std::monostate&) const { + return false; + } + + bool operator()(bool lhs, bool rhs) const { return lhs == rhs; } + + bool operator()(bool, int64_t) const { return false; } + + bool operator()(bool, uint64_t) const { return false; } + + bool operator()(bool, absl::string_view) const { return false; } + + bool operator()(int64_t, bool) const { return false; } + + bool operator()(int64_t lhs, int64_t rhs) const { return lhs == rhs; } + + bool operator()(int64_t, uint64_t) const { return false; } + + bool operator()(int64_t, absl::string_view) const { return false; } + + bool operator()(uint64_t, bool) const { return false; } + + bool operator()(uint64_t, int64_t) const { return false; } + + bool operator()(uint64_t lhs, uint64_t rhs) const { return lhs == rhs; } + + bool operator()(uint64_t, absl::string_view) const { return false; } + + bool operator()(absl::string_view, bool) const { return false; } + + bool operator()(absl::string_view, int64_t) const { return false; } + + bool operator()(absl::string_view, uint64_t) const { return false; } + + bool operator()(absl::string_view lhs, absl::string_view rhs) const { + return lhs == rhs; + } + + bool operator()(const common_internal::WildcardType&, + const common_internal::WildcardType&) const { + return true; + } + + template + std::enable_if_t< + !std::is_same_v, common_internal::WildcardType> && + !std::is_same_v, std::monostate>, + bool> + operator()(const common_internal::WildcardType&, const T&) const { + return false; + } + + template + std::enable_if_t< + !std::is_same_v, common_internal::WildcardType> && + !std::is_same_v, std::monostate>, + bool> + operator()(const T&, const common_internal::WildcardType&) const { + return false; + } +}; + +struct AttributeQualifierLess { + bool operator()(const std::monostate&, const std::monostate&) const { + return false; + } + + template + std::enable_if_t, std::monostate>, + bool> + operator()(const std::monostate&, const T&) const { + return true; + } + + template + std::enable_if_t, std::monostate>, + bool> + operator()(const T&, const std::monostate&) const { + return false; + } + + bool operator()(bool lhs, bool rhs) const { return lhs < rhs; } + + bool operator()(bool, int64_t) const { return false; } + + bool operator()(bool, uint64_t) const { return false; } + + bool operator()(bool, absl::string_view) const { return false; } + + bool operator()(int64_t, bool) const { return true; } + + bool operator()(int64_t lhs, int64_t rhs) const { return lhs < rhs; } + + bool operator()(int64_t, uint64_t) const { return true; } + + bool operator()(int64_t, absl::string_view) const { return true; } + + bool operator()(uint64_t, bool) const { return true; } + + bool operator()(uint64_t, int64_t) const { return false; } + + bool operator()(uint64_t lhs, uint64_t rhs) const { return lhs < rhs; } + + bool operator()(uint64_t, absl::string_view) const { return true; } + + bool operator()(absl::string_view, bool) const { return true; } + + bool operator()(absl::string_view, int64_t) const { return false; } + + bool operator()(absl::string_view, uint64_t) const { return false; } + + bool operator()(absl::string_view lhs, absl::string_view rhs) const { + return lhs < rhs; + } + + bool operator()(const common_internal::WildcardType&, + const common_internal::WildcardType&) const { + return false; + } + + template + std::enable_if_t< + !std::is_same_v, common_internal::WildcardType> && + !std::is_same_v, std::monostate>, + bool> + operator()(const common_internal::WildcardType&, const T&) const { + return false; + } + + template + std::enable_if_t< + !std::is_same_v, common_internal::WildcardType> && + !std::is_same_v, std::monostate>, + bool> + operator()(const T&, const common_internal::WildcardType&) const { + return true; + } +}; + +} // namespace + +bool operator==(const AttributeQualifier& lhs, const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierEqualTo{}, + common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifier& lhs, const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); +} + +bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), + common_internal::AsVariant(rhs)); } } // namespace cel diff --git a/base/attribute.h b/base/attribute.h index 69dcaf161..a0877fc96 100644 --- a/base/attribute.h +++ b/base/attribute.h @@ -15,47 +15,95 @@ #ifndef THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_H_ #define THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_H_ +#include #include #include #include #include #include +#include #include +#include "absl/base/attributes.h" +#include "absl/base/macros.h" +#include "absl/functional/overload.h" +#include "absl/log/absl_check.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" +#include "absl/types/optional_ref.h" #include "absl/types/span.h" -#include "absl/types/variant.h" #include "base/kind.h" namespace cel { -// AttributeQualifier represents a segment in -// attribute resolutuion path. A segment can be qualified by values of +namespace common_internal { +class AttributeMatcherNode; +struct WildcardType {}; +using AttributeQualifierVariant = + std::variant; +using AttributeQualifierPatternVariant = + std::variant; +using AttributeQualifierViewVariant = + std::variant; +} // namespace common_internal + +class AttributeQualifier; +class AttributeQualifierPattern; +class Attribute; +class AttributePattern; +class AttributeQualifierView; + +namespace common_internal { +[[nodiscard]] +const AttributeQualifierVariant& AsVariant( + const AttributeQualifier& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +AttributeQualifierVariant&& AsVariant( + AttributeQualifier&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +const AttributeQualifierPatternVariant& AsVariant( + const AttributeQualifierPattern& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +AttributeQualifierPatternVariant&& AsVariant( + AttributeQualifierPattern&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +const AttributeQualifierViewVariant& AsVariant( + const AttributeQualifierView& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +[[nodiscard]] +AttributeQualifierViewVariant&& AsVariant( + AttributeQualifierView&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND); +} // namespace common_internal + +// AttributeQualifier represents a segment in the +// attribute resolution path. A segment can be qualified by values of // following types: string/int64_t/uint64_t/bool. -class AttributeQualifier final { - private: - struct ComparatorVisitor; - - using Variant = absl::variant; - +class AttributeQualifier { public: static AttributeQualifier OfInt(int64_t value) { - return AttributeQualifier(absl::in_place_type, std::move(value)); + return AttributeQualifier(std::in_place_type, std::move(value)); } static AttributeQualifier OfUint(uint64_t value) { - return AttributeQualifier(absl::in_place_type, std::move(value)); + return AttributeQualifier(std::in_place_type, std::move(value)); + } + + static AttributeQualifier OfString(const char* value) { + return OfString(absl::string_view(value)); } static AttributeQualifier OfString(std::string value) { - return AttributeQualifier(absl::in_place_type, + return AttributeQualifier(std::in_place_type, std::move(value)); } + static AttributeQualifier OfString(absl::string_view value) { + return AttributeQualifier(std::in_place_type, + std::string(value)); + } + static AttributeQualifier OfBool(bool value) { - return AttributeQualifier(absl::in_place_type, std::move(value)); + return AttributeQualifier(std::in_place_type, std::move(value)); } AttributeQualifier() = default; @@ -68,55 +116,126 @@ class AttributeQualifier final { Kind kind() const; - // Family of Get... methods. Return values if requested type matches the - // stored one. - absl::optional GetInt64Key() const { - return absl::holds_alternative(value_) - ? absl::optional(absl::get<1>(value_)) - : absl::nullopt; + [[nodiscard]] + std::string ToString() const; + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetInt64Key() const { return AsInt(); } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetUint64Key() const { return AsUint(); } + + ABSL_DEPRECATED("Use AsString") + std::optional GetStringKey() const { + if (auto string = AsString(); string.has_value()) { + return *string; + } + return std::nullopt; } - absl::optional GetUint64Key() const { - return absl::holds_alternative(value_) - ? absl::optional(absl::get<2>(value_)) - : absl::nullopt; + ABSL_DEPRECATE_AND_INLINE() + std::optional GetBoolKey() const { return AsBool(); } + + explicit operator bool() const { + return !std::holds_alternative(value_); } - absl::optional GetStringKey() const { - return absl::holds_alternative(value_) - ? absl::optional(absl::get<3>(value_)) - : absl::nullopt; + [[nodiscard]] + bool IsBool() const { + return std::holds_alternative(value_); } - absl::optional GetBoolKey() const { - return absl::holds_alternative(value_) - ? absl::optional(absl::get<4>(value_)) - : absl::nullopt; + [[nodiscard]] + bool IsInt() const { + return std::holds_alternative(value_); } - bool operator==(const AttributeQualifier& other) const { - return IsMatch(other); + [[nodiscard]] + bool IsUint() const { + return std::holds_alternative(value_); } - bool operator<(const AttributeQualifier& other) const; + [[nodiscard]] + bool IsString() const { + return std::holds_alternative(value_); + } - bool IsMatch(absl::string_view other_key) const { - absl::optional key = GetStringKey(); - return (key.has_value() && key.value() == other_key); + [[nodiscard]] + bool GetBool() const { + ABSL_DCHECK(IsBool()); + return std::get(value_); } - absl::StatusOr AsString() const; + [[nodiscard]] + int64_t GetInt() const { + ABSL_DCHECK(IsInt()); + return std::get(value_); + } - private: - friend class Attribute; - friend struct ComparatorVisitor; + [[nodiscard]] + uint64_t GetUint() const { + ABSL_DCHECK(IsUint()); + return std::get(value_); + } - template - AttributeQualifier(absl::in_place_type_t in_place_type, T&& value) - : value_(in_place_type, std::forward(value)) {} + [[nodiscard]] + const std::string& GetString() const { + ABSL_DCHECK(IsString()); + return std::get(value_); + } + + [[nodiscard]] + std::optional AsBool() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + [[nodiscard]] + std::optional AsInt() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsUint() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + absl::optional_ref AsString() const { + if (const auto* value = std::get_if(&value_); + value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] bool IsMatch(const AttributeQualifier& other) const; + [[nodiscard]] + bool IsMatch(absl::string_view other_key) const; + + private: + friend const common_internal::AttributeQualifierVariant& + common_internal::AsVariant(const AttributeQualifier& qualifier); + friend common_internal::AttributeQualifierVariant&& + common_internal::AsVariant(AttributeQualifier&& qualifier); + + template + explicit AttributeQualifier(std::in_place_type_t in_place_type, + Args&&... args) + : value_(in_place_type, std::forward(args)...) {} + + using Variant = common_internal::AttributeQualifierVariant; + // The previous implementation of Attribute preserved all value // instances, regardless of whether they are supported in this context or not. // We represented unsupported types by using the first alternative and thus @@ -124,58 +243,580 @@ class AttributeQualifier final { Variant value_; }; -// AttributeQualifierPattern matches a segment in -// attribute resolutuion path. AttributeQualifierPattern is capable of -// matching path elements of types string/int64/uint64/bool. -class AttributeQualifierPattern final { +class AttributeQualifierView { + public: + static AttributeQualifierView OfInt(int64_t value) { + return AttributeQualifierView(std::in_place_type, + std::move(value)); + } + + static AttributeQualifierView OfUint(uint64_t value) { + return AttributeQualifierView(std::in_place_type, + std::move(value)); + } + + static AttributeQualifierView OfString(const char* value) { + return OfString(absl::string_view(value)); + } + + static AttributeQualifierView OfString(absl::string_view value) { + return AttributeQualifierView(std::in_place_type, + std::move(value)); + } + + static AttributeQualifierView OfString(std::string&&) = delete; + + static AttributeQualifierView OfBool(bool value) { + return AttributeQualifierView(std::in_place_type, std::move(value)); + } + + AttributeQualifierView() = default; + AttributeQualifierView(const AttributeQualifierView&) = default; + AttributeQualifierView& operator=(const AttributeQualifierView&) = default; + + // NOLINTNEXTLINE(google-explicit-constructor) + AttributeQualifierView( + const AttributeQualifier& other ABSL_ATTRIBUTE_LIFETIME_BOUND) + : value_(std::visit( + absl::Overload( + [](std::monostate value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](bool value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](int64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](uint64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](const std::string& value) -> Variant { + return Variant(std::in_place_type, value); + }), + common_internal::AsVariant(other))) {} + + // NOLINTNEXTLINE(google-explicit-constructor) + AttributeQualifierView& operator=( + const AttributeQualifier& other ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return *this = AttributeQualifierView(other); + } + + AttributeQualifierView& operator=(AttributeQualifier&&) = delete; + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetInt64Key() const { return AsInt(); } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetUint64Key() const { return AsUint(); } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetStringKey() const { return AsString(); } + + ABSL_DEPRECATE_AND_INLINE() + std::optional GetBoolKey() const { return AsBool(); } + + [[nodiscard]] + bool IsBool() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsInt() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsUint() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsString() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool GetBool() const { + ABSL_DCHECK(IsBool()); + return std::get(value_); + } + + [[nodiscard]] + int64_t GetInt() const { + ABSL_DCHECK(IsInt()); + return std::get(value_); + } + + [[nodiscard]] + uint64_t GetUint() const { + ABSL_DCHECK(IsUint()); + return std::get(value_); + } + + [[nodiscard]] + absl::string_view GetString() const { + ABSL_DCHECK(IsString()); + return std::get(value_); + } + + [[nodiscard]] + std::optional AsBool() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsInt() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsUint() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsString() const { + if (const auto* value = std::get_if(&value_); + value != nullptr) { + return *value; + } + return std::nullopt; + } + private: - // Qualifier value. If not set, treated as wildcard. - std::optional value_; + friend const common_internal::AttributeQualifierViewVariant& + common_internal::AsVariant(const AttributeQualifierView& qualifier); + friend common_internal::AttributeQualifierViewVariant&& + common_internal::AsVariant(AttributeQualifierView&& qualifier); - explicit AttributeQualifierPattern(std::optional value) - : value_(std::move(value)) {} + using Variant = common_internal::AttributeQualifierViewVariant; + template + AttributeQualifierView(std::in_place_type_t in_place_type, T&& value) + : value_(in_place_type, std::forward(value)) {} + + Variant value_; +}; + +// AttributeQualifierPattern matches a segment in +// attribute resolution path. AttributeQualifierPattern is capable of +// matching path elements of types string/int64/uint64/bool. +class AttributeQualifierPattern { public: static AttributeQualifierPattern OfInt(int64_t value) { - return AttributeQualifierPattern(AttributeQualifier::OfInt(value)); + return AttributeQualifierPattern(std::in_place_type, value); } static AttributeQualifierPattern OfUint(uint64_t value) { - return AttributeQualifierPattern(AttributeQualifier::OfUint(value)); + return AttributeQualifierPattern(std::in_place_type, value); + } + + static AttributeQualifierPattern OfString(const char* value) { + return OfString(absl::string_view(value)); } static AttributeQualifierPattern OfString(std::string value) { - return AttributeQualifierPattern( - AttributeQualifier::OfString(std::move(value))); + return AttributeQualifierPattern(std::in_place_type, + std::move(value)); + } + + static AttributeQualifierPattern OfString(absl::string_view value) { + return AttributeQualifierPattern(std::in_place_type, + std::string(value)); } static AttributeQualifierPattern OfBool(bool value) { - return AttributeQualifierPattern(AttributeQualifier::OfBool(value)); + return AttributeQualifierPattern(std::in_place_type, value); + } + + ABSL_DEPRECATE_AND_INLINE() + static AttributeQualifierPattern CreateWildcard() { return Wildcard(); } + + static AttributeQualifierPattern Wildcard() { + return AttributeQualifierPattern(std::in_place_type); + } + + // NOLINTNEXTLINE(google-explicit-constructor) + AttributeQualifierPattern(const AttributeQualifier& value) + : value_(std::visit( + absl::Overload( + [](std::monostate value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](bool value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](int64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](uint64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](const std::string& value) -> Variant { + return Variant(std::in_place_type, value); + }), + common_internal::AsVariant(value))) {} + + // NOLINTNEXTLINE(google-explicit-constructor) + AttributeQualifierPattern(AttributeQualifier&& value) + : value_(std::visit( + absl::Overload( + [](std::monostate value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](bool value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](int64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](uint64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](std::string&& value) -> Variant { + return Variant(std::in_place_type, + std::move(value)); + }), + common_internal::AsVariant(std::move(value)))) {} + + explicit operator bool() const { + return !std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsBool() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsInt() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsUint() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsString() const { + return std::holds_alternative(value_); + } + + [[nodiscard]] + bool IsWildcard() const { + return std::holds_alternative(value_); } - static AttributeQualifierPattern CreateWildcard() { - return AttributeQualifierPattern(std::nullopt); + [[nodiscard]] + bool GetBool() const { + ABSL_DCHECK(IsBool()); + return std::get(value_); } - explicit AttributeQualifierPattern(AttributeQualifier qualifier) - : AttributeQualifierPattern( - std::optional(std::move(qualifier))) {} + [[nodiscard]] + int64_t GetInt() const { + ABSL_DCHECK(IsInt()); + return std::get(value_); + } - bool IsWildcard() const { return !value_.has_value(); } + [[nodiscard]] + uint64_t GetUint() const { + ABSL_DCHECK(IsUint()); + return std::get(value_); + } - bool IsMatch(const AttributeQualifier& qualifier) const { - if (IsWildcard()) return true; - return value_.value() == qualifier; + [[nodiscard]] + const std::string& GetString() const { + ABSL_DCHECK(IsString()); + return std::get(value_); + } + + [[nodiscard]] + std::optional AsBool() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; } - bool IsMatch(absl::string_view other_key) const { - if (!value_.has_value()) return true; - return value_->IsMatch(other_key); + [[nodiscard]] + std::optional AsInt() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional AsUint() const { + if (const auto* value = std::get_if(&value_); value != nullptr) { + return *value; + } + return std::nullopt; } + + [[nodiscard]] + absl::optional_ref AsString() const { + if (const auto* value = std::get_if(&value_); + value != nullptr) { + return *value; + } + return std::nullopt; + } + + [[nodiscard]] + std::optional ToQualifier() const { + return std::visit( + absl::Overload( + [](std::monostate) -> std::optional { + return AttributeQualifier(); + }, + [](bool value) -> std::optional { + return AttributeQualifier::OfBool(value); + }, + [](int64_t value) -> std::optional { + return AttributeQualifier::OfInt(value); + }, + [](uint64_t value) -> std::optional { + return AttributeQualifier::OfUint(value); + }, + [](const std::string& value) -> std::optional { + return AttributeQualifier::OfString(value); + }, + [](common_internal::WildcardType value) + -> std::optional { return std::nullopt; }), + value_); + } + + [[nodiscard]] + std::optional ToQualifierView() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return std::visit( + absl::Overload( + [](std::monostate) -> std::optional { + return AttributeQualifierView(); + }, + [](bool value) -> std::optional { + return AttributeQualifierView::OfBool(value); + }, + [](int64_t value) -> std::optional { + return AttributeQualifierView::OfInt(value); + }, + [](uint64_t value) -> std::optional { + return AttributeQualifierView::OfUint(value); + }, + [](const std::string& value) + -> std::optional { + return AttributeQualifierView::OfString(value); + }, + [](common_internal::WildcardType value) + -> std::optional { + return std::nullopt; + }), + value_); + } + + [[nodiscard]] + bool IsMatch(const AttributeQualifier& qualifier) const; + + [[nodiscard]] + bool IsMatch(absl::string_view other_key) const; + + private: + friend const common_internal::AttributeQualifierPatternVariant& + common_internal::AsVariant(const AttributeQualifierPattern& qualifier); + friend common_internal::AttributeQualifierPatternVariant&& + common_internal::AsVariant(AttributeQualifierPattern&& qualifier); + + using Variant = common_internal::AttributeQualifierPatternVariant; + using WildcardType = common_internal::WildcardType; + + template + explicit AttributeQualifierPattern(std::in_place_type_t in_place_type, + Args&&... args) + : value_(in_place_type, std::forward(args)...) {} + + // Qualifier value. If not set, treated as wildcard. + common_internal::AttributeQualifierPatternVariant value_; }; +[[nodiscard]] +bool operator==(const AttributeQualifier& lhs, const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator==(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs); + +[[nodiscard]] +inline bool operator!=(const AttributeQualifier& lhs, + const AttributeQualifier& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator!=(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +bool operator<(const AttributeQualifier& lhs, const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs); + +[[nodiscard]] +bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator<(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs); + +[[nodiscard]] +bool operator<(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs); + +[[nodiscard]] +bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs); + +[[nodiscard]] +bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs); + +namespace common_internal { + +[[nodiscard]] +inline const AttributeQualifierVariant& AsVariant( + const AttributeQualifier& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return qualifier.value_; +} + +[[nodiscard]] +inline AttributeQualifierVariant&& AsVariant( + AttributeQualifier&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return std::move(qualifier.value_); +} + +[[nodiscard]] +inline const AttributeQualifierPatternVariant& AsVariant( + const AttributeQualifierPattern& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return qualifier.value_; +} + +[[nodiscard]] +inline AttributeQualifierPatternVariant&& AsVariant( + AttributeQualifierPattern&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return std::move(qualifier.value_); +} + +[[nodiscard]] +inline const AttributeQualifierViewVariant& AsVariant( + const AttributeQualifierView& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return qualifier.value_; +} + +[[nodiscard]] +inline AttributeQualifierViewVariant&& AsVariant( + AttributeQualifierView&& qualifier ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return std::move(qualifier.value_); +} + +} // namespace common_internal + // Attribute represents resolved attribute path. -class Attribute final { +class Attribute { public: explicit Attribute(std::string variable_name) : Attribute(std::move(variable_name), {}) {} @@ -197,7 +838,7 @@ class Attribute final { bool operator<(const Attribute& other) const; - const absl::StatusOr AsString() const; + absl::StatusOr AsString() const; private: struct Impl final { @@ -218,7 +859,7 @@ class Attribute final { // - field selection; // - map lookup by key; // - list access by index. -class AttributePattern final { +class AttributePattern { public: // MatchType enum specifies how closely pattern is matching the attribute: enum class MatchType { @@ -271,7 +912,7 @@ struct FieldSpecifier { std::string name; }; -using SelectQualifier = absl::variant; +using SelectQualifier = std::variant; } // namespace cel diff --git a/base/attribute_test.cc b/base/attribute_test.cc new file mode 100644 index 000000000..d4b36f9f5 --- /dev/null +++ b/base/attribute_test.cc @@ -0,0 +1,80 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "base/attribute.h" + +#include + +#include "absl/strings/string_view.h" +#include "internal/testing.h" + +namespace cel { +namespace { + +using ::testing::Optional; + +TEST(AttributeQualifierView, Bool) { + AttributeQualifierView qualifier = AttributeQualifierView::OfBool(true); + EXPECT_TRUE(qualifier.IsBool()); + EXPECT_FALSE(qualifier.IsInt()); + EXPECT_FALSE(qualifier.IsUint()); + EXPECT_FALSE(qualifier.IsString()); + EXPECT_TRUE(qualifier.GetBool()); + EXPECT_THAT(qualifier.AsBool(), Optional(true)); + EXPECT_THAT(qualifier.AsInt(), std::nullopt); + EXPECT_THAT(qualifier.AsUint(), std::nullopt); + EXPECT_THAT(qualifier.AsString(), std::nullopt); +} + +TEST(AttributeQualifierView, Int) { + AttributeQualifierView qualifier = AttributeQualifierView::OfInt(1); + EXPECT_FALSE(qualifier.IsBool()); + EXPECT_TRUE(qualifier.IsInt()); + EXPECT_FALSE(qualifier.IsUint()); + EXPECT_FALSE(qualifier.IsString()); + EXPECT_EQ(qualifier.GetInt(), 1); + EXPECT_THAT(qualifier.AsBool(), std::nullopt); + EXPECT_THAT(qualifier.AsInt(), Optional(1)); + EXPECT_THAT(qualifier.AsUint(), std::nullopt); + EXPECT_THAT(qualifier.AsString(), std::nullopt); +} + +TEST(AttributeQualifierView, Uint) { + AttributeQualifierView qualifier = AttributeQualifierView::OfUint(1); + EXPECT_FALSE(qualifier.IsBool()); + EXPECT_FALSE(qualifier.IsInt()); + EXPECT_TRUE(qualifier.IsUint()); + EXPECT_FALSE(qualifier.IsString()); + EXPECT_EQ(qualifier.GetUint(), 1); + EXPECT_THAT(qualifier.AsBool(), std::nullopt); + EXPECT_THAT(qualifier.AsInt(), std::nullopt); + EXPECT_THAT(qualifier.AsUint(), Optional(1)); + EXPECT_THAT(qualifier.AsString(), std::nullopt); +} + +TEST(AttributeQualifierView, String) { + AttributeQualifierView qualifier = AttributeQualifierView::OfString("foo"); + EXPECT_FALSE(qualifier.IsBool()); + EXPECT_FALSE(qualifier.IsInt()); + EXPECT_FALSE(qualifier.IsUint()); + EXPECT_TRUE(qualifier.IsString()); + EXPECT_EQ(qualifier.GetString(), "foo"); + EXPECT_THAT(qualifier.AsBool(), std::nullopt); + EXPECT_THAT(qualifier.AsInt(), std::nullopt); + EXPECT_THAT(qualifier.AsUint(), std::nullopt); + EXPECT_THAT(qualifier.AsString(), Optional(absl::string_view("foo"))); +} + +} // namespace +} // namespace cel diff --git a/eval/public/cel_attribute_test.cc b/eval/public/cel_attribute_test.cc index b72189332..424dae92f 100644 --- a/eval/public/cel_attribute_test.cc +++ b/eval/public/cel_attribute_test.cc @@ -52,7 +52,7 @@ TEST(CelAttributeQualifierTest, TestBoolAccess) { EXPECT_FALSE(qualifier.GetUint64Key().has_value()); EXPECT_TRUE(qualifier.GetBoolKey().has_value()); EXPECT_THAT(qualifier.GetBoolKey().value(), Eq(true)); - EXPECT_THAT(qualifier.AsString(), IsOkAndHolds("true")); + EXPECT_THAT(qualifier.ToString(), "true"); } TEST(CelAttributeQualifierTest, TestInt64Access) { @@ -64,7 +64,7 @@ TEST(CelAttributeQualifierTest, TestInt64Access) { EXPECT_TRUE(qualifier.GetInt64Key().has_value()); EXPECT_THAT(qualifier.GetInt64Key().value(), Eq(-1)); - EXPECT_THAT(qualifier.AsString(), IsOkAndHolds("-1")); + EXPECT_THAT(qualifier.ToString(), "-1"); } TEST(CelAttributeQualifierTest, TestUint64Access) { @@ -76,7 +76,7 @@ TEST(CelAttributeQualifierTest, TestUint64Access) { EXPECT_TRUE(qualifier.GetUint64Key().has_value()); EXPECT_THAT(qualifier.GetUint64Key().value(), Eq(1UL)); - EXPECT_THAT(qualifier.AsString(), IsOkAndHolds("1")); + EXPECT_THAT(qualifier.ToString(), "1"); } TEST(CelAttributeQualifierTest, TestStringAccess) { @@ -89,7 +89,7 @@ TEST(CelAttributeQualifierTest, TestStringAccess) { EXPECT_TRUE(qualifier.GetStringKey().has_value()); EXPECT_THAT(qualifier.GetStringKey().value(), Eq("test")); - EXPECT_THAT(qualifier.AsString(), IsOkAndHolds("test")); + EXPECT_THAT(qualifier.ToString(), "test"); } void TestAllInequalities(const CelAttributeQualifier& qualifier) { diff --git a/extensions/protobuf/internal/qualify.cc b/extensions/protobuf/internal/qualify.cc index 37ad30011..1d53dac0d 100644 --- a/extensions/protobuf/internal/qualify.cc +++ b/extensions/protobuf/internal/qualify.cc @@ -339,11 +339,7 @@ absl::StatusOr ProtoQualifyState::CheckMapIn qualifier)); if (!value_ref.has_value()) { - std::string key_string; - absl::StatusOr key_string_or = qualifier.AsString(); - if (key_string_or.ok()) { - key_string = *key_string_or; - } + std::string key_string = qualifier.ToString(); return runtime_internal::CreateNoSuchKeyError(key_string); } return std::move(value_ref).value(); From 2ff1fbd238f09bf625f705f38c1cb365dae90e41 Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 29 Sep 2026 10:20:57 -0700 Subject: [PATCH 244/350] Introduce RBTree implementation tailored for `proto2::Arena` PiperOrigin-RevId: 990385693 --- internal/BUILD | 21 ++ internal/arena_tree.cc | 345 ++++++++++++++++++++ internal/arena_tree.h | 390 ++++++++++++++++++++++ internal/arena_tree_test.cc | 627 ++++++++++++++++++++++++++++++++++++ 4 files changed, 1383 insertions(+) create mode 100644 internal/arena_tree.cc create mode 100644 internal/arena_tree.h create mode 100644 internal/arena_tree_test.cc diff --git a/internal/BUILD b/internal/BUILD index 189853323..d6781b538 100644 --- a/internal/BUILD +++ b/internal/BUILD @@ -42,6 +42,27 @@ cc_test( ], ) +cc_library( + name = "arena_tree", + srcs = ["arena_tree.cc"], + hdrs = ["arena_tree.h"], + deps = [ + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "arena_tree_test", + srcs = ["arena_tree_test.cc"], + deps = [ + ":arena_tree", + ":testing", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "new", srcs = ["new.cc"], diff --git a/internal/arena_tree.cc b/internal/arena_tree.cc new file mode 100644 index 000000000..8db8fde6c --- /dev/null +++ b/internal/arena_tree.cc @@ -0,0 +1,345 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/arena_tree.h" + +namespace cel::internal { + +namespace { + +void ArenaTreeRotateLeft(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* tmp = ArenaTreeNodeGetRight(elem); + if (ArenaTreeNodeSetRight(elem, ArenaTreeNodeGetLeft(tmp)) != nullptr) { + ArenaTreeNodeSetParent(ArenaTreeNodeGetLeft(tmp), elem); + } + ArenaTreeNodeBase* parent = ArenaTreeNodeGetParent(elem); + ArenaTreeNodeSetParent(tmp, parent); + if (parent != nullptr) { + if (elem == ArenaTreeNodeGetLeft(parent)) { + ArenaTreeNodeSetLeft(parent, tmp); + } else { + ArenaTreeNodeSetRight(parent, tmp); + } + } else { + *head = tmp; + } + ArenaTreeNodeSetLeft(tmp, elem); + ArenaTreeNodeSetParent(elem, tmp); +} + +void ArenaTreeRotateRight(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* tmp = ArenaTreeNodeGetLeft(elem); + if (ArenaTreeNodeSetLeft(elem, ArenaTreeNodeGetRight(tmp)) != nullptr) { + ArenaTreeNodeSetParent(ArenaTreeNodeGetRight(tmp), elem); + } + ArenaTreeNodeBase* parent = ArenaTreeNodeGetParent(elem); + ArenaTreeNodeSetParent(tmp, parent); + if (parent != nullptr) { + if (elem == ArenaTreeNodeGetLeft(parent)) { + ArenaTreeNodeSetLeft(parent, tmp); + } else { + ArenaTreeNodeSetRight(parent, tmp); + } + } else { + *head = tmp; + } + ArenaTreeNodeSetRight(tmp, elem); + ArenaTreeNodeSetParent(elem, tmp); +} + +void ArenaTreeRemoveColor(ArenaTreeNodeBase** head, ArenaTreeNodeBase* parent, + ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* tmp; + while ((elem == nullptr || + ArenaTreeNodeGetColor(elem) == ArenaTreeNodeColor::kBlack) && + elem != *head) { + if (ArenaTreeNodeGetLeft(parent) == elem) { + tmp = ArenaTreeNodeGetRight(parent); + if (ArenaTreeNodeGetColor(tmp) == ArenaTreeNodeColor::kRed) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kRed); + ArenaTreeRotateLeft(head, parent); + tmp = ArenaTreeNodeGetRight(parent); + } + if ((ArenaTreeNodeGetLeft(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetLeft(tmp)) == + ArenaTreeNodeColor::kBlack) && + (ArenaTreeNodeGetRight(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetRight(tmp)) == + ArenaTreeNodeColor::kBlack)) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kRed); + elem = parent; + parent = ArenaTreeNodeGetParent(elem); + } else { + if (ArenaTreeNodeGetRight(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetRight(tmp)) == + ArenaTreeNodeColor::kBlack) { + ArenaTreeNodeBase* left; + if ((left = ArenaTreeNodeGetLeft(tmp)) != nullptr) { + ArenaTreeNodeSetColor(left, ArenaTreeNodeColor::kBlack); + } + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kRed); + ArenaTreeRotateRight(head, tmp); + tmp = ArenaTreeNodeGetRight(parent); + } + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeGetColor(parent)); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + if (ArenaTreeNodeGetRight(tmp) != nullptr) { + ArenaTreeNodeSetColor(ArenaTreeNodeGetRight(tmp), + ArenaTreeNodeColor::kBlack); + } + ArenaTreeRotateLeft(head, parent); + elem = *head; + break; + } + } else { + tmp = ArenaTreeNodeGetLeft(parent); + if (ArenaTreeNodeGetColor(tmp) == ArenaTreeNodeColor::kRed) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kRed); + ArenaTreeRotateRight(head, parent); + tmp = ArenaTreeNodeGetLeft(parent); + } + if ((ArenaTreeNodeGetLeft(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetLeft(tmp)) == + ArenaTreeNodeColor::kBlack) && + (ArenaTreeNodeGetRight(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetRight(tmp)) == + ArenaTreeNodeColor::kBlack)) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kRed); + elem = parent; + parent = ArenaTreeNodeGetParent(elem); + } else { + if (ArenaTreeNodeGetLeft(tmp) == nullptr || + ArenaTreeNodeGetColor(ArenaTreeNodeGetLeft(tmp)) == + ArenaTreeNodeColor::kBlack) { + ArenaTreeNodeBase* right; + if ((right = ArenaTreeNodeGetRight(tmp)) != nullptr) { + ArenaTreeNodeSetColor(right, ArenaTreeNodeColor::kBlack); + } + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kRed); + ArenaTreeRotateLeft(head, tmp); + tmp = ArenaTreeNodeGetLeft(parent); + } + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeGetColor(parent)); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + if (ArenaTreeNodeGetLeft(tmp) != nullptr) { + ArenaTreeNodeSetColor(ArenaTreeNodeGetLeft(tmp), + ArenaTreeNodeColor::kBlack); + } + ArenaTreeRotateRight(head, parent); + elem = *head; + break; + } + } + } + if (elem != nullptr) { + ArenaTreeNodeSetColor(elem, ArenaTreeNodeColor::kBlack); + } +} + +} // namespace + +const ArenaTreeNodeBase* ArenaTreeNext(const ArenaTreeNodeBase* node) { + if (node != nullptr) { + const ArenaTreeNodeBase* right = ArenaTreeNodeGetRight(node); + if (right != nullptr) { + node = right; + const ArenaTreeNodeBase* left; + while ((left = ArenaTreeNodeGetLeft(node)) != nullptr) { + node = left; + } + } else { + const ArenaTreeNodeBase* parent = ArenaTreeNodeGetParent(node); + if (parent != nullptr && node == ArenaTreeNodeGetLeft(parent)) { + node = parent; + } else { + while (parent != nullptr && ArenaTreeNodeGetRight(parent) == node) { + node = ArenaTreeNodeGetParent(node); + parent = ArenaTreeNodeGetParent(node); + } + node = parent; + } + } + } + return node; +} + +const ArenaTreeNodeBase* ArenaTreePrev(const ArenaTreeNodeBase* node) { + if (node != nullptr) { + const ArenaTreeNodeBase* left = ArenaTreeNodeGetLeft(node); + if (left != nullptr) { + node = left; + const ArenaTreeNodeBase* right; + while ((right = ArenaTreeNodeGetRight(node)) != nullptr) { + node = right; + } + } else { + const ArenaTreeNodeBase* parent = ArenaTreeNodeGetParent(node); + if (parent != nullptr && node == ArenaTreeNodeGetRight(parent)) { + node = parent; + } else { + while (parent != nullptr && ArenaTreeNodeGetLeft(parent) == node) { + node = ArenaTreeNodeGetParent(node); + parent = ArenaTreeNodeGetParent(node); + } + node = parent; + } + } + } + return node; +} + +const ArenaTreeNodeBase* ArenaTreeMin(const ArenaTreeNodeBase* node) { + const ArenaTreeNodeBase* tmp = node; + const ArenaTreeNodeBase* parent = nullptr; + while (tmp != nullptr) { + parent = tmp; + tmp = ArenaTreeNodeGetLeft(tmp); + } + return parent; +} + +const ArenaTreeNodeBase* ArenaTreeMax(const ArenaTreeNodeBase* node) { + const ArenaTreeNodeBase* tmp = node; + const ArenaTreeNodeBase* parent = nullptr; + while (tmp != nullptr) { + parent = tmp; + tmp = ArenaTreeNodeGetRight(tmp); + } + return parent; +} + +void ArenaTreeRemove(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* child; + ArenaTreeNodeBase* parent; + ArenaTreeNodeBase* const old = elem; + ArenaTreeNodeColor color; + if (ArenaTreeNodeGetLeft(elem) == nullptr) { + child = ArenaTreeNodeGetRight(elem); + } else if (ArenaTreeNodeGetRight(elem) == nullptr) { + child = ArenaTreeNodeGetLeft(elem); + } else { + ArenaTreeNodeBase* left; + elem = ArenaTreeNodeGetRight(elem); + while ((left = ArenaTreeNodeGetLeft(elem)) != nullptr) { + elem = left; + } + child = ArenaTreeNodeGetRight(elem); + parent = ArenaTreeNodeGetParent(elem); + color = ArenaTreeNodeGetColor(elem); + if (child != nullptr) { + ArenaTreeNodeSetParent(child, parent); + } + if (parent != nullptr) { + if (ArenaTreeNodeGetLeft(parent) == elem) { + ArenaTreeNodeSetLeft(parent, child); + } else { + ArenaTreeNodeSetRight(parent, child); + } + } else { + *head = child; + } + if (ArenaTreeNodeGetParent(elem) == old) { + parent = elem; + } + ArenaTreeNodeCopy(elem, old); + ArenaTreeNodeBase* old_parent = ArenaTreeNodeGetParent(old); + if (old_parent != nullptr) { + if (ArenaTreeNodeGetLeft(old_parent) == old) { + ArenaTreeNodeSetLeft(old_parent, elem); + } else { + ArenaTreeNodeSetRight(old_parent, elem); + } + } else { + *head = elem; + } + ArenaTreeNodeSetParent(ArenaTreeNodeGetLeft(old), elem); + if (ArenaTreeNodeGetRight(old) != nullptr) { + ArenaTreeNodeSetParent(ArenaTreeNodeGetRight(old), elem); + } + goto color; + } + parent = ArenaTreeNodeGetParent(elem); + color = ArenaTreeNodeGetColor(elem); + if (child != nullptr) { + ArenaTreeNodeSetParent(child, parent); + } + if (parent != nullptr) { + if (ArenaTreeNodeGetLeft(parent) == elem) { + ArenaTreeNodeSetLeft(parent, child); + } else { + ArenaTreeNodeSetRight(parent, child); + } + } else { + *head = child; + } +color: + if (color == ArenaTreeNodeColor::kBlack) { + ArenaTreeRemoveColor(head, parent, child); + } + ArenaTreeNodeClear(old); +} + +void ArenaTreeInsertColor(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem) { + ArenaTreeNodeBase* parent; + ArenaTreeNodeBase* grandparent; + ArenaTreeNodeBase* tmp; + while ((parent = ArenaTreeNodeGetParent(elem)) != nullptr && + ArenaTreeNodeGetColor(parent) == ArenaTreeNodeColor::kRed) { + grandparent = ArenaTreeNodeGetParent(parent); + if (parent == ArenaTreeNodeGetLeft(grandparent)) { + tmp = ArenaTreeNodeGetRight(grandparent); + if (tmp != nullptr && + ArenaTreeNodeGetColor(tmp) == ArenaTreeNodeColor::kRed) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(grandparent, ArenaTreeNodeColor::kRed); + elem = grandparent; + continue; + } + if (ArenaTreeNodeGetRight(parent) == elem) { + ArenaTreeRotateLeft(head, parent); + tmp = parent; + parent = elem; + elem = tmp; + } + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(grandparent, ArenaTreeNodeColor::kRed); + ArenaTreeRotateRight(head, grandparent); + } else { + tmp = ArenaTreeNodeGetLeft(grandparent); + if (tmp != nullptr && + ArenaTreeNodeGetColor(tmp) == ArenaTreeNodeColor::kRed) { + ArenaTreeNodeSetColor(tmp, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(grandparent, ArenaTreeNodeColor::kRed); + elem = grandparent; + continue; + } + if (ArenaTreeNodeGetLeft(parent) == elem) { + ArenaTreeRotateRight(head, parent); + tmp = parent; + parent = elem; + elem = tmp; + } + ArenaTreeNodeSetColor(parent, ArenaTreeNodeColor::kBlack); + ArenaTreeNodeSetColor(grandparent, ArenaTreeNodeColor::kRed); + ArenaTreeRotateLeft(head, grandparent); + } + } + ArenaTreeNodeSetColor(*head, ArenaTreeNodeColor::kBlack); +} + +} // namespace cel::internal diff --git a/internal/arena_tree.h b/internal/arena_tree.h new file mode 100644 index 000000000..05f0e74bd --- /dev/null +++ b/internal/arena_tree.h @@ -0,0 +1,390 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ArenaTree is a low level implementation of an RBTree tailored for use with +// google::protobuf::Arena. It avoids needing to deal with using a custom Allocator with +// std::set, std::map, absl::btree_set, or absl::btree_map, where the options +// are manually construct on the arena to avoid registering the destructor +// (technically undefined behavior) or registering the destructor and running it +// even though it is a no-op at runtime. Using absl::btree_set or +// absl::btree_map also does not preserve pointer stability and they have a +// higher overhead for small containers. +// +// Note that you should not use this unless you are really sure you need to. In +// most cases you should prefer std::set, std::map, absl::btree_set, or +// absl::btree_map. + +#ifndef THIRD_PARTY_CEL_CPP_INTERNAL_ARENA_TREE_H_ +#define THIRD_PARTY_CEL_CPP_INTERNAL_ARENA_TREE_H_ + +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "google/protobuf/arena.h" + +namespace cel::internal { + +enum class ArenaTreeNodeColor : uintptr_t { + kBlack = 0, + kRed = 1, +}; + +class ArenaTreeNodeBase; + +[[nodiscard]] +const ArenaTreeNodeBase* ArenaTreeNext(const ArenaTreeNodeBase* node); + +[[nodiscard]] +const ArenaTreeNodeBase* ArenaTreePrev(const ArenaTreeNodeBase* node); + +[[nodiscard]] +const ArenaTreeNodeBase* ArenaTreeMin(const ArenaTreeNodeBase* node); + +[[nodiscard]] +const ArenaTreeNodeBase* ArenaTreeMax(const ArenaTreeNodeBase* node); + +template +using IsDerivedFromArenaTreeNodeBase = std::conjunction< + std::is_base_of, + std::negation>>>; + +template +constexpr bool kIsDerivedFromArenaTreeNodeBase = + IsDerivedFromArenaTreeNodeBase::value; + +template +using EnableIfDerivedFromArenaTreeNodeBase = + std::enable_if_t, U>; + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreeNext(T* node) { + return static_cast(const_cast( + (ArenaTreeNext)(static_cast(node)))); +} + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreePrev(T* node) { + return static_cast(const_cast( + (ArenaTreePrev)(static_cast(node)))); +} + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreeMin(T* node) { + return static_cast(const_cast( + (ArenaTreeMin)(static_cast(node)))); +} + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreeMax(T* node) { + return static_cast(const_cast( + (ArenaTreeMax)(static_cast(node)))); +} + +[[nodiscard]] +ArenaTreeNodeBase* ArenaTreeNodeGetParent(const ArenaTreeNodeBase* node); + +void ArenaTreeNodeSetParent(ArenaTreeNodeBase* node, ArenaTreeNodeBase* parent); + +[[nodiscard]] +ArenaTreeNodeColor ArenaTreeNodeGetColor(const ArenaTreeNodeBase* node); + +void ArenaTreeNodeSetColor(ArenaTreeNodeBase* node, ArenaTreeNodeColor color); + +void ArenaTreeNodeSet(ArenaTreeNodeBase* node, ArenaTreeNodeBase* parent); + +void ArenaTreeNodeCopy(ArenaTreeNodeBase* dst, const ArenaTreeNodeBase* src); + +void ArenaTreeNodeClear(ArenaTreeNodeBase* node); + +[[nodiscard]] +ArenaTreeNodeBase* ArenaTreeNodeGetLeft(const ArenaTreeNodeBase* node); + +ArenaTreeNodeBase* ArenaTreeNodeSetLeft(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* left); + +[[nodiscard]] +ArenaTreeNodeBase* ArenaTreeNodeGetRight(const ArenaTreeNodeBase* node); + +ArenaTreeNodeBase* ArenaTreeNodeSetRight(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* right); + +class ArenaTreeNodeBase { + private: + uintptr_t parent_and_color_ = 0; + ArenaTreeNodeBase* left_ = nullptr; + ArenaTreeNodeBase* right_ = nullptr; + + friend ArenaTreeNodeBase* ArenaTreeNodeGetParent( + const ArenaTreeNodeBase* node); + friend void ArenaTreeNodeSetParent(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* parent); + friend ArenaTreeNodeColor ArenaTreeNodeGetColor( + const ArenaTreeNodeBase* node); + friend void ArenaTreeNodeSetColor(ArenaTreeNodeBase* node, + ArenaTreeNodeColor color); + friend void ArenaTreeNodeSet(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* parent); + friend void ArenaTreeNodeClear(ArenaTreeNodeBase* node); + friend void ArenaTreeNodeCopy(ArenaTreeNodeBase* dst, + const ArenaTreeNodeBase* src); + friend ArenaTreeNodeBase* ArenaTreeNodeGetLeft(const ArenaTreeNodeBase* node); + friend ArenaTreeNodeBase* ArenaTreeNodeSetLeft(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* left); + friend ArenaTreeNodeBase* ArenaTreeNodeGetRight( + const ArenaTreeNodeBase* node); + friend ArenaTreeNodeBase* ArenaTreeNodeSetRight(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* right); +}; + +[[nodiscard]] +inline ArenaTreeNodeBase* ArenaTreeNodeGetParent( + const ArenaTreeNodeBase* node) { + return reinterpret_cast(node->parent_and_color_ & + ~uintptr_t{1}); +} + +inline void ArenaTreeNodeSetParent(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* parent) { + node->parent_and_color_ = + static_cast(ArenaTreeNodeGetColor(node)) | + reinterpret_cast(parent); +} + +[[nodiscard]] +inline ArenaTreeNodeColor ArenaTreeNodeGetColor(const ArenaTreeNodeBase* node) { + return static_cast(node->parent_and_color_ & + uintptr_t{1}); +} + +inline void ArenaTreeNodeSetColor(ArenaTreeNodeBase* node, + ArenaTreeNodeColor color) { + node->parent_and_color_ = + reinterpret_cast(ArenaTreeNodeGetParent(node)) | + static_cast(color); +} + +inline void ArenaTreeNodeSet(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* parent) { + node->parent_and_color_ = reinterpret_cast(parent) | + static_cast(ArenaTreeNodeColor::kRed); + node->left_ = node->right_ = nullptr; +} + +inline void ArenaTreeNodeClear(ArenaTreeNodeBase* node) { + node->parent_and_color_ = 0; + node->left_ = node->right_ = nullptr; +} + +inline void ArenaTreeNodeCopy(ArenaTreeNodeBase* dst, + const ArenaTreeNodeBase* src) { + dst->parent_and_color_ = src->parent_and_color_; + dst->left_ = src->left_; + dst->right_ = src->right_; +} + +[[nodiscard]] +inline ArenaTreeNodeBase* ArenaTreeNodeGetLeft(const ArenaTreeNodeBase* node) { + return node->left_; +} + +inline ArenaTreeNodeBase* ArenaTreeNodeSetLeft(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* left) { + return node->left_ = left; +} + +[[nodiscard]] +inline ArenaTreeNodeBase* ArenaTreeNodeGetRight(const ArenaTreeNodeBase* node) { + return node->right_; +} + +inline ArenaTreeNodeBase* ArenaTreeNodeSetRight(ArenaTreeNodeBase* node, + ArenaTreeNodeBase* right) { + return node->right_ = right; +} + +void ArenaTreeRemove(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem); + +template +[[nodiscard]] +inline EnableIfDerivedFromArenaTreeNodeBase ArenaTreeRemove(T** head, + T* elem) { + ArenaTreeNodeBase* head_base = *head; + (ArenaTreeRemove)(&head_base, static_cast(elem)); + *head = static_cast(head_base); +} + +void ArenaTreeInsertColor(ArenaTreeNodeBase** head, ArenaTreeNodeBase* elem); + +template +class ArenaTreeNode; + +template +[[nodiscard]] +const T& ArenaTreeNodeGetValue(const ArenaTreeNode* node); + +template +class ArenaTreeNode : public ArenaTreeNodeBase { + public: + template + explicit ArenaTreeNode(Args&&... args) + : ArenaTreeNodeBase(), value_(std::forward(args)...) {} + + private: + template + friend const U& ArenaTreeNodeGetValue(const ArenaTreeNode* node); + + T value_; +}; + +template +[[nodiscard]] inline const T& ArenaTreeNodeGetValue( + const ArenaTreeNode* node) { + return node->value_; +} + +template +class ArenaTreeNodeCrtp : public ArenaTreeNodeBase { + public: + using ArenaTreeNodeBase::ArenaTreeNodeBase; +}; + +template +[[nodiscard]] inline const T& ArenaTreeNodeGetValue( + const ArenaTreeNodeCrtp* node) { + return *static_cast(node); +} + +template +[[nodiscard]] +T* ArenaTreeInsert(T** head, T* elem, const Compare& compare) { + T* tmp = *head; + T* parent = nullptr; + int diff = 0; + while (tmp != nullptr) { + parent = tmp; + diff = std::invoke(compare, (ArenaTreeNodeGetValue)(elem), + (ArenaTreeNodeGetValue)(parent)); + if (diff < 0) { + tmp = static_cast((ArenaTreeNodeGetLeft)(tmp)); + } else if (diff > 0) { + tmp = static_cast((ArenaTreeNodeGetRight)(tmp)); + } else { + return tmp; + } + } + (ArenaTreeNodeSet)(elem, parent); + if (parent != nullptr) { + if (diff < 0) { + (ArenaTreeNodeSetLeft)(parent, elem); + } else { + (ArenaTreeNodeSetRight)(parent, elem); + } + } else { + *head = elem; + } + ArenaTreeNodeBase* head_base = *head; + (ArenaTreeInsertColor)(&head_base, elem); + *head = static_cast(head_base); + return elem; +} + +template +struct ArenaTreeNodeConstructor { + template + void operator()(Args&&... args) const { + ABSL_DCHECK(*out == nullptr); + *out = google::protobuf::Arena::Create(arena, std::forward(args)...); + } + + google::protobuf::Arena* const arena; + T** out; +}; + +template +[[nodiscard]] +std::pair ArenaTreeLazyEmplace(google::protobuf::Arena* arena, T** head, + const K& key, const Compare& compare, + Emplacer&& emplacer) { + T* tmp = *head; + T* parent = nullptr; + int diff = 0; + while (tmp != nullptr) { + parent = tmp; + diff = std::invoke(compare, key, (ArenaTreeNodeGetValue)(parent)); + if (diff < 0) { + tmp = static_cast((ArenaTreeNodeGetLeft)(tmp)); + } else if (diff > 0) { + tmp = static_cast((ArenaTreeNodeGetRight)(tmp)); + } else { + return {tmp, false}; + } + } + T* elem = nullptr; + ArenaTreeNodeConstructor constructor{ + .arena = arena, + .out = &elem, + }; + std::invoke(std::forward(emplacer), + static_cast&>(constructor)); + ABSL_DCHECK(elem != nullptr); + (ArenaTreeNodeSet)(elem, parent); + if (parent != nullptr) { + if (diff < 0) { + (ArenaTreeNodeSetLeft)(parent, elem); + } else { + (ArenaTreeNodeSetRight)(parent, elem); + } + } else { + *head = elem; + } + ArenaTreeNodeBase* head_base = *head; + (ArenaTreeInsertColor)(&head_base, elem); + *head = static_cast(head_base); + return {elem, true}; +} + +template +[[nodiscard]] +const T* ArenaTreeFind(const T* head, const K& key, const Compare& compare) { + const T* tmp = head; + while (tmp != nullptr) { + int diff = std::invoke(compare, key, (ArenaTreeNodeGetValue)(tmp)); + if (diff < 0) { + tmp = static_cast((ArenaTreeNodeGetLeft)(tmp)); + } else if (diff > 0) { + tmp = static_cast((ArenaTreeNodeGetRight)(tmp)); + } else { + return tmp; + } + } + return nullptr; +} + +template +[[nodiscard]] +T* ArenaTreeFind(T* head, const K& key, const Compare& compare) { + return const_cast( + (ArenaTreeFind)(static_cast(head), key, compare)); +} + +} // namespace cel::internal + +#endif // THIRD_PARTY_CEL_CPP_INTERNAL_ARENA_TREE_H_ diff --git a/internal/arena_tree_test.cc b/internal/arena_tree_test.cc new file mode 100644 index 000000000..9f6dfdbf7 --- /dev/null +++ b/internal/arena_tree_test.cc @@ -0,0 +1,627 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/arena_tree.h" + +#include +#include +#include +#include +#include + +#include "internal/testing.h" +#include "google/protobuf/arena.h" + +namespace cel::internal { +namespace { + +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::IsNull; + +using TestNode = ArenaTreeNode; + +struct TestNodeCompare { + int operator()(int lhs, int rhs) const { + if (lhs < rhs) { + return -1; + } + if (lhs > rhs) { + return 1; + } + return 0; + } +}; + +struct TestNodeCrtp : ArenaTreeNodeCrtp { + explicit TestNodeCrtp(int value) : ArenaTreeNodeCrtp(), value(value) {} + + int value; +}; + +struct TestNodeCrtpCompare { + int operator()(int lhs, int rhs) const { + if (lhs < rhs) { + return -1; + } + if (lhs > rhs) { + return 1; + } + return 0; + } + + int operator()(int lhs, const TestNodeCrtp& rhs) const { + return (*this)(lhs, rhs.value); + } + + int operator()(const TestNodeCrtp& lhs, int rhs) const { + return (*this)(lhs.value, rhs); + } + + int operator()(const TestNodeCrtp& lhs, const TestNodeCrtp& rhs) const { + return (*this)(lhs.value, rhs.value); + } +}; + +template +int VerifySubtree(const T* node, const T* expected_parent, const T* lower_bound, + const T* upper_bound, const Compare& compare) { + if (node == nullptr) { + return 1; + } + EXPECT_EQ(ArenaTreeNodeGetParent(node), expected_parent); + if (lower_bound != nullptr) { + EXPECT_LT(compare(ArenaTreeNodeGetValue(lower_bound), + ArenaTreeNodeGetValue(node)), + 0); + } + if (upper_bound != nullptr) { + EXPECT_LT(compare(ArenaTreeNodeGetValue(node), + ArenaTreeNodeGetValue(upper_bound)), + 0); + } + const auto* left = static_cast(ArenaTreeNodeGetLeft(node)); + const auto* right = static_cast(ArenaTreeNodeGetRight(node)); + if (ArenaTreeNodeGetColor(node) == ArenaTreeNodeColor::kRed) { + if (left != nullptr) { + EXPECT_EQ(ArenaTreeNodeGetColor(left), ArenaTreeNodeColor::kBlack); + } + if (right != nullptr) { + EXPECT_EQ(ArenaTreeNodeGetColor(right), ArenaTreeNodeColor::kBlack); + } + } + int left_black_height = VerifySubtree(left, node, lower_bound, node, compare); + int right_black_height = + VerifySubtree(right, node, node, upper_bound, compare); + EXPECT_EQ(left_black_height, right_black_height); + return left_black_height + + (ArenaTreeNodeGetColor(node) == ArenaTreeNodeColor::kBlack ? 1 : 0); +} + +template +std::vector VerifyTree(T* head, const Compare& compare) { + if (head == nullptr) { + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + return {}; + } + EXPECT_EQ(ArenaTreeNodeGetColor(head), ArenaTreeNodeColor::kBlack); + VerifySubtree(static_cast(head), static_cast(nullptr), + static_cast(nullptr), static_cast(nullptr), + compare); + + std::vector forward; + for (const T* curr = ArenaTreeMin(static_cast(head)); + curr != nullptr; curr = static_cast(ArenaTreeNext(curr))) { + forward.push_back(curr); + EXPECT_EQ(ArenaTreeFind(static_cast(head), + ArenaTreeNodeGetValue(curr), compare), + curr); + EXPECT_EQ(ArenaTreeFind(head, ArenaTreeNodeGetValue(curr), compare), curr); + } + EXPECT_FALSE(forward.empty()); + EXPECT_EQ(forward.front(), ArenaTreeMin(head)); + EXPECT_EQ(forward.back(), ArenaTreeMax(head)); + + std::vector backward; + for (T* curr = ArenaTreeMax(head); curr != nullptr; + curr = ArenaTreePrev(curr)) { + backward.push_back(curr); + } + std::reverse(backward.begin(), backward.end()); + EXPECT_EQ(forward.size(), backward.size()); + for (size_t i = 0; i < forward.size(); ++i) { + EXPECT_EQ(forward[i], backward[i]); + } + + std::vector values; + values.reserve(forward.size()); + for (const T* node : forward) { + if constexpr (std::is_same_v) { + values.push_back(ArenaTreeNodeGetValue(node)); + } else { + values.push_back(ArenaTreeNodeGetValue(node).value); + } + } + return values; +} + +TEST(ArenaTree, Empty) { + TestNode* head = nullptr; + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); +} + +TEST(ArenaTree, Single) { + TestNode* head = nullptr; + auto node1 = std::make_unique(1); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreePrev(head), nullptr); + EXPECT_EQ(ArenaTreeNext(head), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node1.get()); + EXPECT_EQ(ArenaTreeFind(head, 1, TestNodeCompare{}), node1.get()); + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + EXPECT_THAT(ArenaTreeFind(head, 1, TestNodeCompare{}), IsNull()); +} + +TEST(ArenaTree, Couple) { + TestNode* head = nullptr; + auto node1 = std::make_unique(1); + auto node2 = std::make_unique(2); + + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreePrev(node1.get()), nullptr); + EXPECT_EQ(ArenaTreePrev(node2.get()), node1.get()); + EXPECT_EQ(ArenaTreeNext(node1.get()), node2.get()); + EXPECT_EQ(ArenaTreeNext(node2.get()), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_EQ(ArenaTreeMin(head), node2.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node2.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreePrev(node1.get()), nullptr); + EXPECT_EQ(ArenaTreePrev(node2.get()), node1.get()); + EXPECT_EQ(ArenaTreeNext(node1.get()), node2.get()); + EXPECT_EQ(ArenaTreeNext(node2.get()), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node2.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node1.get()); + + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); +} + +TEST(ArenaTree, CrtpEmpty) { + TestNodeCrtp* head = nullptr; + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); +} + +TEST(ArenaTree, CrtpSingle) { + TestNodeCrtp* head = nullptr; + auto node1 = std::make_unique(1); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCrtpCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreePrev(head), nullptr); + EXPECT_EQ(ArenaTreeNext(head), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node1.get()); + EXPECT_EQ(ArenaTreeFind(head, 1, TestNodeCrtpCompare{}), node1.get()); + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + EXPECT_THAT(ArenaTreeFind(head, 1, TestNodeCrtpCompare{}), IsNull()); +} + +TEST(ArenaTree, CrtpCouple) { + TestNodeCrtp* head = nullptr; + auto node1 = std::make_unique(1); + auto node2 = std::make_unique(2); + + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCrtpCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCrtpCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreePrev(node1.get()), nullptr); + EXPECT_EQ(ArenaTreePrev(node2.get()), node1.get()); + EXPECT_EQ(ArenaTreeNext(node1.get()), node2.get()); + EXPECT_EQ(ArenaTreeNext(node2.get()), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_EQ(ArenaTreeMin(head), node2.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node2.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); + + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCrtpCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCrtpCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreePrev(node1.get()), nullptr); + EXPECT_EQ(ArenaTreePrev(node2.get()), node1.get()); + EXPECT_EQ(ArenaTreeNext(node1.get()), node2.get()); + EXPECT_EQ(ArenaTreeNext(node2.get()), nullptr); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node2.get()); + + ArenaTreeRemove(&head, node2.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_EQ(ArenaTreeMin(head), node1.get()); + EXPECT_EQ(ArenaTreeMax(head), node1.get()); + + ArenaTreeRemove(&head, node1.get()); + EXPECT_THAT(ArenaTreePrev(head), IsNull()); + EXPECT_THAT(ArenaTreeNext(head), IsNull()); + EXPECT_THAT(ArenaTreeMin(head), IsNull()); + EXPECT_THAT(ArenaTreeMax(head), IsNull()); +} + +TEST(ArenaTree, InsertDuplicateReturnsExistingNode) { + TestNode* head = nullptr; + auto node2 = std::make_unique(2); + auto node1 = std::make_unique(1); + auto node3 = std::make_unique(3); + auto dup2 = std::make_unique(2); + auto dup1 = std::make_unique(1); + auto dup3 = std::make_unique(3); + + EXPECT_EQ(ArenaTreeInsert(&head, node2.get(), TestNodeCompare{}), + node2.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node1.get(), TestNodeCompare{}), + node1.get()); + EXPECT_EQ(ArenaTreeInsert(&head, node3.get(), TestNodeCompare{}), + node3.get()); + + EXPECT_EQ(ArenaTreeInsert(&head, dup2.get(), TestNodeCompare{}), node2.get()); + EXPECT_EQ(ArenaTreeInsert(&head, dup1.get(), TestNodeCompare{}), node1.get()); + EXPECT_EQ(ArenaTreeInsert(&head, dup3.get(), TestNodeCompare{}), node3.get()); + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), ElementsAre(1, 2, 3)); +} + +TEST(ArenaTree, LazyEmplace) { + google::protobuf::Arena arena; + TestNode* head = nullptr; + + int emplace_calls = 0; + auto emplace_val = [&](int key) { + return ArenaTreeLazyEmplace(&arena, &head, key, TestNodeCompare{}, + [&](const auto& ctor) { + ++emplace_calls; + ctor(key); + }); + }; + + auto [node2, inserted2] = emplace_val(2); + EXPECT_TRUE(inserted2); + EXPECT_EQ(ArenaTreeNodeGetValue(node2), 2); + EXPECT_EQ(emplace_calls, 1); + + auto [node1, inserted1] = emplace_val(1); + EXPECT_TRUE(inserted1); + EXPECT_EQ(ArenaTreeNodeGetValue(node1), 1); + EXPECT_EQ(emplace_calls, 2); + + auto [node3, inserted3] = emplace_val(3); + EXPECT_TRUE(inserted3); + EXPECT_EQ(ArenaTreeNodeGetValue(node3), 3); + EXPECT_EQ(emplace_calls, 3); + + // Duplicate keys should not invoke the emplacer. + auto [dup2, dup_inserted2] = emplace_val(2); + EXPECT_FALSE(dup_inserted2); + EXPECT_EQ(dup2, node2); + EXPECT_EQ(emplace_calls, 3); + + auto [dup1, dup_inserted1] = emplace_val(1); + EXPECT_FALSE(dup_inserted1); + EXPECT_EQ(dup1, node1); + EXPECT_EQ(emplace_calls, 3); + + auto [dup3, dup_inserted3] = emplace_val(3); + EXPECT_FALSE(dup_inserted3); + EXPECT_EQ(dup3, node3); + EXPECT_EQ(emplace_calls, 3); + + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), ElementsAre(1, 2, 3)); +} + +TEST(ArenaTree, CrtpLazyEmplace) { + google::protobuf::Arena arena; + TestNodeCrtp* head = nullptr; + + int emplace_calls = 0; + auto emplace_val = [&](int key) { + return ArenaTreeLazyEmplace(&arena, &head, key, TestNodeCrtpCompare{}, + [&](const auto& ctor) { + ++emplace_calls; + ctor(key); + }); + }; + + auto [node2, inserted2] = emplace_val(2); + EXPECT_TRUE(inserted2); + EXPECT_EQ(ArenaTreeNodeGetValue(node2).value, 2); + EXPECT_EQ(emplace_calls, 1); + + auto [node1, inserted1] = emplace_val(1); + EXPECT_TRUE(inserted1); + EXPECT_EQ(ArenaTreeNodeGetValue(node1).value, 1); + EXPECT_EQ(emplace_calls, 2); + + auto [node3, inserted3] = emplace_val(3); + EXPECT_TRUE(inserted3); + EXPECT_EQ(ArenaTreeNodeGetValue(node3).value, 3); + EXPECT_EQ(emplace_calls, 3); + + auto [dup2, dup_inserted2] = emplace_val(2); + EXPECT_FALSE(dup_inserted2); + EXPECT_EQ(dup2, node2); + EXPECT_EQ(emplace_calls, 3); + + EXPECT_THAT(VerifyTree(head, TestNodeCrtpCompare{}), ElementsAre(1, 2, 3)); +} + +TEST(ArenaTree, FindLeftRightAndMissing) { + google::protobuf::Arena arena; + TestNode* head = nullptr; + for (int key : {8, 4, 12, 2, 6, 10, 14}) { + auto [node, inserted] = + ArenaTreeLazyEmplace(&arena, &head, key, TestNodeCompare{}, + [key](const auto& ctor) { ctor(key); }); + EXPECT_TRUE(inserted); + } + + const TestNode* const_head = head; + for (int key : {2, 4, 6, 8, 10, 12, 14}) { + TestNode* found = ArenaTreeFind(head, key, TestNodeCompare{}); + ASSERT_NE(found, nullptr); + EXPECT_EQ(ArenaTreeNodeGetValue(found), key); + EXPECT_EQ(ArenaTreeFind(const_head, key, TestNodeCompare{}), found); + } + for (int missing : {1, 3, 5, 7, 9, 11, 13, 15}) { + EXPECT_THAT(ArenaTreeFind(head, missing, TestNodeCompare{}), IsNull()); + EXPECT_THAT(ArenaTreeFind(const_head, missing, TestNodeCompare{}), + IsNull()); + } +} + +TEST(ArenaTree, InsertAndRemoveAscendingAndDescending) { + constexpr int kCount = 64; + std::vector> nodes; + nodes.reserve(kCount); + for (int i = 0; i < kCount; ++i) { + nodes.push_back(std::make_unique(i)); + } + + // Insert ascending, remove ascending. + TestNode* head = nullptr; + for (int i = 0; i < kCount; ++i) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[i].get(), TestNodeCompare{}), + nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + for (int i = 0; i < kCount; ++i) { + ArenaTreeRemove(&head, nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + EXPECT_THAT(head, IsNull()); + + // Insert descending, remove descending. + for (int i = kCount - 1; i >= 0; --i) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[i].get(), TestNodeCompare{}), + nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + for (int i = kCount - 1; i >= 0; --i) { + ArenaTreeRemove(&head, nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + EXPECT_THAT(head, IsNull()); + + // Insert ascending, remove descending. + for (int i = 0; i < kCount; ++i) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[i].get(), TestNodeCompare{}), + nodes[i].get()); + } + VerifyTree(head, TestNodeCompare{}); + for (int i = kCount - 1; i >= 0; --i) { + ArenaTreeRemove(&head, nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + EXPECT_THAT(head, IsNull()); + + // Insert descending, remove ascending. + for (int i = kCount - 1; i >= 0; --i) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[i].get(), TestNodeCompare{}), + nodes[i].get()); + } + VerifyTree(head, TestNodeCompare{}); + for (int i = 0; i < kCount; ++i) { + ArenaTreeRemove(&head, nodes[i].get()); + VerifyTree(head, TestNodeCompare{}); + } + EXPECT_THAT(head, IsNull()); +} + +TEST(ArenaTree, InsertZigZagAndRemoveInternalNodes) { + constexpr int kCount = 63; + std::vector> nodes; + nodes.reserve(kCount); + for (int i = 0; i < kCount; ++i) { + nodes.push_back(std::make_unique(i)); + } + + // Insert in a zig-zag pattern (alternating low and high values) to exercise + // Left-Right and Right-Left triangle rotations in ArenaTreeInsertColor. + TestNode* head = nullptr; + int lo = 0; + int hi = kCount - 1; + while (lo <= hi) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[lo].get(), TestNodeCompare{}), + nodes[lo].get()); + VerifyTree(head, TestNodeCompare{}); + if (lo < hi) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[hi].get(), TestNodeCompare{}), + nodes[hi].get()); + VerifyTree(head, TestNodeCompare{}); + } + ++lo; + --hi; + } + + // Repeatedly remove left child of root, right child of root, and root itself + // to cover 2-child removals where old_parent is null, old is a left child, + // and old is a right child. + int step = 0; + while (head != nullptr) { + TestNode* target = head; + if (step % 3 == 0 && ArenaTreeNodeGetLeft(head) != nullptr) { + target = static_cast(ArenaTreeNodeGetLeft(head)); + } else if (step % 3 == 1 && ArenaTreeNodeGetRight(head) != nullptr) { + target = static_cast(ArenaTreeNodeGetRight(head)); + } + ArenaTreeRemove(&head, target); + VerifyTree(head, TestNodeCompare{}); + ++step; + } +} + +TEST(ArenaTree, RemoveSingleLeftAndSingleRightChildCases) { + // Specifically exercise ArenaTreeRemove when the removed node has only a left + // child or only a right child (as root, left child of parent, and right child + // of parent). + std::vector> nodes; + for (int i = 0; i < 10; ++i) { + nodes.push_back(std::make_unique(i)); + } + + // Case 1: Node with only a left child as left child of parent and right child + // of parent. + TestNode* head = nullptr; + for (int idx : {5, 2, 8, 1, 7}) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[idx].get(), TestNodeCompare{}), + nodes[idx].get()); + } + // Node 2 has only left child 1; Node 8 has only left child 7. + ArenaTreeRemove(&head, nodes[2].get()); + VerifyTree(head, TestNodeCompare{}); + ArenaTreeRemove(&head, nodes[8].get()); + VerifyTree(head, TestNodeCompare{}); + while (head != nullptr) { + ArenaTreeRemove(&head, head); + } + + // Case 2: Node with only a right child as left child of parent and right + // child of parent. + for (int idx : {5, 2, 8, 3, 9}) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[idx].get(), TestNodeCompare{}), + nodes[idx].get()); + } + // Node 2 has only right child 3; Node 8 has only right child 9. + ArenaTreeRemove(&head, nodes[2].get()); + VerifyTree(head, TestNodeCompare{}); + ArenaTreeRemove(&head, nodes[8].get()); + VerifyTree(head, TestNodeCompare{}); + while (head != nullptr) { + ArenaTreeRemove(&head, head); + } +} + +TEST(ArenaTree, AllPermutationsInsertAndRemove) { + constexpr int kPermSize = 6; + std::vector> nodes; + nodes.reserve(kPermSize); + std::vector insert_order(kPermSize); + for (int i = 0; i < kPermSize; ++i) { + nodes.push_back(std::make_unique(i)); + insert_order[i] = i; + } + + do { + TestNode* head = nullptr; + for (int idx : insert_order) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[idx].get(), TestNodeCompare{}), + nodes[idx].get()); + } + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), + ElementsAre(0, 1, 2, 3, 4, 5)); + + // Remove in insertion order. + for (int idx : insert_order) { + ArenaTreeRemove(&head, nodes[idx].get()); + } + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), IsEmpty()); + + // Re-insert and remove in reverse insertion order. + for (int idx : insert_order) { + EXPECT_EQ(ArenaTreeInsert(&head, nodes[idx].get(), TestNodeCompare{}), + nodes[idx].get()); + } + for (int i = kPermSize - 1; i >= 0; --i) { + ArenaTreeRemove(&head, nodes[insert_order[i]].get()); + } + EXPECT_THAT(VerifyTree(head, TestNodeCompare{}), IsEmpty()); + } while (std::next_permutation(insert_order.begin(), insert_order.end())); +} + +} // namespace +} // namespace cel::internal From 74994b848be7cda7048f394b5b70709b17a63289 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 29 Sep 2026 11:14:17 -0700 Subject: [PATCH 245/350] Add special constant instruction in the main interpret loop. Consolidate error reporting and move error check outside of Eval loop. PiperOrigin-RevId: 990420160 --- eval/compiler/BUILD | 1 - eval/compiler/constant_folding.cc | 4 +- eval/compiler/constant_folding_test.cc | 98 ++++++------ eval/compiler/flat_expr_builder.cc | 4 +- .../flat_expr_builder_extensions_test.cc | 29 ++-- .../regex_precompilation_optimization.cc | 11 +- eval/eval/BUILD | 6 +- eval/eval/compiler_constant_step.cc | 6 - eval/eval/compiler_constant_step.h | 23 --- eval/eval/compiler_constant_step_test.cc | 40 +++-- eval/eval/const_value_step.h | 7 - eval/eval/create_list_step_test.cc | 4 +- eval/eval/evaluator_core.cc | 130 +++++++++++++++- eval/eval/evaluator_core.h | 143 ++++++++++++------ eval/eval/evaluator_core_test.cc | 48 ++++++ eval/eval/function_step_test.cc | 15 +- eval/eval/lazy_init_step_test.cc | 7 +- 17 files changed, 379 insertions(+), 197 deletions(-) diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index dbd8d9c8b..6916c96a5 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -360,7 +360,6 @@ cc_test( "//base:ast", "//common:expr", "//common:value", - "//eval/eval:const_value_step", "//eval/eval:create_list_step", "//eval/eval:create_map_step", "//eval/eval:evaluator_core", diff --git a/eval/compiler/constant_folding.cc b/eval/compiler/constant_folding.cc index 93cb35e65..071f16186 100644 --- a/eval/compiler/constant_folding.cc +++ b/eval/compiler/constant_folding.cc @@ -50,7 +50,6 @@ using ::cel::builtin::kOr; using ::cel::builtin::kTernary; using ::cel::runtime_internal::ConvertConstant; using ::google::api::expr::runtime::CreateConstValueDirectStep; -using ::google::api::expr::runtime::CreateConstValueStep; using ::google::api::expr::runtime::EvaluationListener; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionPath; @@ -243,8 +242,7 @@ absl::Status ConstantFoldingExtension::OnPostVisit(PlannerContext& context, // Otherwise make a stack machine plan. ExecutionPath new_plan; - new_plan.push_back(ExpressionStep::MakeGenericStep( - CreateConstValueStep(std::move(value)), node.id())); + new_plan.push_back(ExpressionStep::MakeConstant(value, node.id())); return context.ReplaceSubplan(node, std::move(new_plan)); } diff --git a/eval/compiler/constant_folding_test.cc b/eval/compiler/constant_folding_test.cc index 1528f5a5d..8c6fcf54c 100644 --- a/eval/compiler/constant_folding_test.cc +++ b/eval/compiler/constant_folding_test.cc @@ -29,7 +29,6 @@ #include "common/value.h" #include "eval/compiler/flat_expr_builder_extensions.h" #include "eval/compiler/resolver.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/create_list_step.h" #include "eval/eval/create_map_step.h" #include "eval/eval/evaluator_core.h" @@ -58,7 +57,6 @@ using ::cel::runtime_internal::IssueCollector; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::ParsedExpr; using ::google::api::expr::parser::Parse; -using ::google::api::expr::runtime::CreateConstValueStep; using ::google::api::expr::runtime::CreateCreateListStep; using ::google::api::expr::runtime::CreateCreateStructStepForMap; using ::google::api::expr::runtime::ExecutionPath; @@ -116,25 +114,25 @@ TEST_F(UpdatedConstantFoldingTest, SkipsTernary) { program_builder.EnterSubexpression(&call); // condition program_builder.EnterSubexpression(&condition); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(true)), condition.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), condition.id())); program_builder.ExitSubexpression(&condition); // true program_builder.EnterSubexpression(&true_branch); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(true)), true_branch.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), true_branch.id())); program_builder.ExitSubexpression(&true_branch); // false program_builder.EnterSubexpression(&false_branch); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(true)), false_branch.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), false_branch.id())); program_builder.ExitSubexpression(&false_branch); // ternary. - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::NullValue()), call.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::NullValue(), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -179,20 +177,20 @@ TEST_F(UpdatedConstantFoldingTest, SkipsOr) { // left program_builder.EnterSubexpression(&left_condition); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(false)), left_condition.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(false), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(true)), right_condition.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::NullValue()), call.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::NullValue(), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -234,20 +232,20 @@ TEST_F(UpdatedConstantFoldingTest, SkipsAnd) { // left program_builder.EnterSubexpression(&left_condition); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(true)), left_condition.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(false)), right_condition.id())); + program_builder.AddStep(ExpressionStep::MakeConstant(cel::BoolValue(false), + right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::NullValue()), call.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::NullValue(), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; @@ -289,14 +287,14 @@ TEST_F(UpdatedConstantFoldingTest, CreatesList) { // elem one program_builder.EnterSubexpression(&elem_one); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(1L)), elem_one.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(1L), elem_one.id())); program_builder.ExitSubexpression(&elem_one); // elem two program_builder.EnterSubexpression(&elem_two); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(2L)), elem_two.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2L), elem_two.id())); program_builder.ExitSubexpression(&elem_two); // createlist @@ -349,32 +347,32 @@ TEST_F(UpdatedConstantFoldingTest, CreatesLargeList) { // 0 ASSERT_TRUE(program_builder.EnterSubexpression(&elem0) != nullptr); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(1L)), elem0.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(1L), elem0.id())); program_builder.ExitSubexpression(&elem0); // 1 ASSERT_TRUE(program_builder.EnterSubexpression(&elem1)); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(2L)), elem1.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2L), elem1.id())); program_builder.ExitSubexpression(&elem1); // 2 ASSERT_TRUE(program_builder.EnterSubexpression(&elem2) != nullptr); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(3L)), elem2.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(3L), elem2.id())); program_builder.ExitSubexpression(&elem2); // 3 ASSERT_TRUE(program_builder.EnterSubexpression(&elem3) != nullptr); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(4L)), elem3.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(4L), elem3.id())); program_builder.ExitSubexpression(&elem3); // 4 ASSERT_TRUE(program_builder.EnterSubexpression(&elem4) != nullptr); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(5L)), elem4.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(5L), elem4.id())); program_builder.ExitSubexpression(&elem4); // createlist @@ -428,14 +426,14 @@ TEST_F(UpdatedConstantFoldingTest, CreatesMap) { // key program_builder.EnterSubexpression(&key); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(1L)), key.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(1L), key.id())); program_builder.ExitSubexpression(&key); // value program_builder.EnterSubexpression(&value); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(2L)), value.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2L), value.id())); program_builder.ExitSubexpression(&value); // create map @@ -484,14 +482,14 @@ TEST_F(UpdatedConstantFoldingTest, CreatesInvalidMap) { // key program_builder.EnterSubexpression(&key); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::DoubleValue(1.0)), key.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::DoubleValue(1.0), key.id())); program_builder.ExitSubexpression(&key); // value program_builder.EnterSubexpression(&value); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::IntValue(2L)), value.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::IntValue(2L), value.id())); program_builder.ExitSubexpression(&value); // create map @@ -539,20 +537,20 @@ TEST_F(UpdatedConstantFoldingTest, ErrorsOnUnexpectedOrder) { program_builder.EnterSubexpression(&call); // left program_builder.EnterSubexpression(&left_condition); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(true)), left_condition.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::BoolValue(true), left_condition.id())); program_builder.ExitSubexpression(&left_condition); // right program_builder.EnterSubexpression(&right_condition); - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::BoolValue(false)), right_condition.id())); + program_builder.AddStep(ExpressionStep::MakeConstant(cel::BoolValue(false), + right_condition.id())); program_builder.ExitSubexpression(&right_condition); // op // Just a placeholder. - program_builder.AddStep(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::NullValue()), call.id())); + program_builder.AddStep( + ExpressionStep::MakeConstant(cel::NullValue(), call.id())); program_builder.ExitSubexpression(&call); std::shared_ptr arena; diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 677918c7c..7cb94fe45 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -730,8 +730,8 @@ class FlatExprVisitor : public cel::AstVisitor { return; } - AddStep(CreateConstValueStep(std::move(converted_value).value()), - expr.id()); + AddStep(ExpressionStep::MakeConstant(std::move(converted_value).value(), + expr.id())); } struct SlotLookupResult { diff --git a/eval/compiler/flat_expr_builder_extensions_test.cc b/eval/compiler/flat_expr_builder_extensions_test.cc index a1e8ecd4b..db13f6ec6 100644 --- a/eval/compiler/flat_expr_builder_extensions_test.cc +++ b/eval/compiler/flat_expr_builder_extensions_test.cc @@ -55,6 +55,17 @@ using ::testing::ElementsAre; using ::testing::IsEmpty; using ::testing::Optional; +class TestStepLogic : public ExpressionStepLogic { + public: + absl::Status Evaluate(ExecutionFrame* frame) const override { + return absl::OkStatus(); + } +}; + +std::unique_ptr MakeTestStepLogic() { + return std::make_unique(); +} + using Subexpression = ProgramBuilder::Subexpression; class PlannerContextTest : public testing::Test { @@ -94,9 +105,9 @@ struct SimpleTreeSteps { absl::StatusOr InitSimpleTree( const Expr& a, const Expr& b, const Expr& c, ProgramBuilder& program_builder) { - auto a_step = CreateConstValueStep(cel::NullValue()); - auto b_step = CreateConstValueStep(cel::NullValue()); - auto c_step = CreateConstValueStep(cel::NullValue()); + auto a_step = MakeTestStepLogic(); + auto b_step = MakeTestStepLogic(); + auto c_step = MakeTestStepLogic(); SimpleTreeSteps result{a_step.get(), b_step.get(), c_step.get()}; @@ -163,7 +174,7 @@ TEST_F(PlannerContextTest, ReplacePlan) { ExecutionPath new_a; - auto new_a_step = CreateConstValueStep(cel::NullValue()); + auto new_a_step = MakeTestStepLogic(); const ExpressionStepLogic* new_a_step_ptr = new_a_step.get(); new_a.push_back(ExpressionStep::MakeGenericStep(std::move(new_a_step), -1)); @@ -253,10 +264,10 @@ TEST_F(PlannerContextTest, ReplacePlanUpdatesSibling) { ExecutionPath new_b; - auto b1_step = CreateConstValueStep(cel::NullValue()); + auto b1_step = MakeTestStepLogic(); const ExpressionStepLogic* b1_step_ptr = b1_step.get(); new_b.push_back(ExpressionStep::MakeGenericStep(std::move(b1_step), -1)); - auto b2_step = CreateConstValueStep(cel::NullValue()); + auto b2_step = MakeTestStepLogic(); const ExpressionStepLogic* b2_step_ptr = b2_step.get(); new_b.push_back(ExpressionStep::MakeGenericStep(std::move(b2_step), -1)); @@ -302,7 +313,7 @@ TEST_F(PlannerContextTest, AddSubplanStep) { ASSERT_OK_AND_ASSIGN(auto plan_steps, InitSimpleTree(a, b, c, program_builder)); - auto b2_step = CreateConstValueStep(cel::NullValue()); + auto b2_step = MakeTestStepLogic(); const ExpressionStepLogic* b2_step_ptr = b2_step.get(); @@ -331,7 +342,7 @@ TEST_F(PlannerContextTest, AddSubplanStepFailsOnUnknownNode) { ASSERT_THAT(InitSimpleTree(a, b, c, program_builder).status(), IsOk()); - auto b2_step = CreateConstValueStep(cel::NullValue()); + auto b2_step = MakeTestStepLogic(); std::shared_ptr arena; PlannerContext context(env_, resolver_, options_, @@ -478,7 +489,7 @@ TEST_F(ProgramBuilderTest, ExtractWorks) { program_builder.EnterSubexpression(&b); program_builder.ExitSubexpression(&b); - auto a_step = CreateConstValueStep(cel::NullValue()); + auto a_step = MakeTestStepLogic(); program_builder.AddStep( ExpressionStep::MakeGenericStep(std::move(a_step), -1)); program_builder.EnterSubexpression(&c); diff --git a/eval/compiler/regex_precompilation_optimization.cc b/eval/compiler/regex_precompilation_optimization.cc index 46c8817af..fa0d289f2 100644 --- a/eval/compiler/regex_precompilation_optimization.cc +++ b/eval/compiler/regex_precompilation_optimization.cc @@ -194,12 +194,11 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { } else { // otherwise stack-machine program. ExecutionPathView re_plan = context.GetSubplan(re_expr); - if (re_plan.size() == 1 && re_plan[0].IsGenericStep() && - re_plan[0].GetGenericStep()->GetNativeTypeId() == - NativeTypeId::For()) { - constant = - down_cast(re_plan[0].GetGenericStep()) - ->value(); + if (re_plan.size() == 1) { + cel::Value val; + if (GetIfConstant(re_plan[0], val)) { + constant = std::move(val); + } } } diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 1b4de3631..de692b8bf 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -48,6 +48,7 @@ cc_library( "//base:data", "//common:native_type", "//common:value", + "//common:value_kind", "//runtime", "//runtime:activation_interface", "//runtime:runtime_options", @@ -182,7 +183,6 @@ cc_library( deps = [ ":compiler_constant_step", ":direct_expression_step", - ":evaluator_core", "//common:value", "@com_google_absl//absl/status:statusor", ], @@ -732,6 +732,7 @@ cc_test( "//runtime:standard_functions", "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_macros", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -1137,7 +1138,6 @@ cc_library( ":attribute_trail", ":direct_expression_step", ":evaluator_core", - ":expression_step_base", "//common:native_type", "//common:value", "@com_google_absl//absl/status", @@ -1148,7 +1148,9 @@ cc_test( name = "compiler_constant_step_test", srcs = ["compiler_constant_step_test.cc"], deps = [ + ":attribute_trail", ":compiler_constant_step", + ":direct_expression_step", ":evaluator_core", "//common:native_type", "//common:value", diff --git a/eval/eval/compiler_constant_step.cc b/eval/eval/compiler_constant_step.cc index 44a03cecd..c1b46b92e 100644 --- a/eval/eval/compiler_constant_step.cc +++ b/eval/eval/compiler_constant_step.cc @@ -28,10 +28,4 @@ absl::Status DirectCompilerConstantStep::Evaluate( return absl::OkStatus(); } -absl::Status CompilerConstantStep::Evaluate(ExecutionFrame* frame) const { - frame->value_stack().Push(value_); - - return absl::OkStatus(); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/compiler_constant_step.h b/eval/eval/compiler_constant_step.h index 967f6512f..6708e02de 100644 --- a/eval/eval/compiler_constant_step.h +++ b/eval/eval/compiler_constant_step.h @@ -22,8 +22,6 @@ #include "common/value.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" namespace google::api::expr::runtime { @@ -49,27 +47,6 @@ class DirectCompilerConstantStep : public DirectExpressionStep { cel::Value value_; }; -// ExpressionStep implementation that simply pushes a constant value on the -// stack. -// -// Overrides NativeTypeId ()o allow the FlatExprBuilder and extensions to -// inspect the underlying value. -class CompilerConstantStep : public ExpressionStepBase { - public: - explicit CompilerConstantStep(cel::Value value) : value_(std::move(value)) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - const cel::Value& value() const { return value_; } - - private: - cel::Value value_; -}; - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPILER_CONSTANT_STEP_H_ diff --git a/eval/eval/compiler_constant_step_test.cc b/eval/eval/compiler_constant_step_test.cc index 8fb226813..181728479 100644 --- a/eval/eval/compiler_constant_step_test.cc +++ b/eval/eval/compiler_constant_step_test.cc @@ -13,10 +13,10 @@ // limitations under the License. #include "eval/eval/compiler_constant_step.h" -#include - #include "common/native_type.h" #include "common/value.h" +#include "eval/eval/attribute_trail.h" +#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" @@ -30,43 +30,41 @@ namespace google::api::expr::runtime { namespace { -class CompilerConstantStepTest : public testing::Test { +class DirectCompilerConstantStepTest : public testing::Test { public: - CompilerConstantStepTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()), - state_(2, 0, type_provider_, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_) {} + DirectCompilerConstantStepTest() + : type_provider_(cel::internal::GetTestingDescriptorPool()) {} protected: google::protobuf::Arena arena_; cel::runtime_internal::RuntimeTypeProvider type_provider_; - FlatExpressionEvaluatorState state_; cel::Activation empty_activation_; cel::RuntimeOptions options_; }; -TEST_F(CompilerConstantStepTest, Evaluate) { - ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep( - std::make_unique(cel::IntValue(42)))); - - ExecutionFrame frame(path, empty_activation_, options_, state_); +TEST_F(DirectCompilerConstantStepTest, Evaluate) { + ExecutionFrameBase frame(empty_activation_, options_, type_provider_, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena_); + DirectCompilerConstantStep step(cel::IntValue(42), -1); + cel::Value result; + AttributeTrail attr; - ASSERT_OK_AND_ASSIGN(cel::Value result, frame.Evaluate()); + ASSERT_THAT(step.Evaluate(frame, result, attr), absl_testing::IsOk()); EXPECT_EQ(result.GetInt().NativeValue(), 42); } -TEST_F(CompilerConstantStepTest, TypeId) { - CompilerConstantStep step(cel::IntValue(42)); +TEST_F(DirectCompilerConstantStepTest, TypeId) { + DirectCompilerConstantStep step(cel::IntValue(42), -1); - ExpressionStepLogic& abstract_step = step; + const DirectExpressionStep& abstract_step = step; EXPECT_EQ(abstract_step.GetNativeTypeId(), - cel::NativeTypeId::For()); + cel::NativeTypeId::For()); } -TEST_F(CompilerConstantStepTest, Value) { - CompilerConstantStep step(cel::IntValue(42)); +TEST_F(DirectCompilerConstantStepTest, Value) { + DirectCompilerConstantStep step(cel::IntValue(42), -1); EXPECT_EQ(step.value().GetInt().NativeValue(), 42); } diff --git a/eval/eval/const_value_step.h b/eval/eval/const_value_step.h index 8a08b580a..55425ceab 100644 --- a/eval/eval/const_value_step.h +++ b/eval/eval/const_value_step.h @@ -9,7 +9,6 @@ #include "common/value.h" #include "eval/eval/compiler_constant_step.h" #include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { @@ -19,12 +18,6 @@ inline std::unique_ptr CreateConstValueDirectStep( return std::make_unique(std::move(value), id); } -// Factory method for Constant AST node expression step. -inline std::unique_ptr CreateConstValueStep( - cel::Value value) { - return std::make_unique(std::move(value)); -} - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONST_VALUE_STEP_H_ diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index ebb1ecb41..2faf1f11e 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -80,8 +80,8 @@ absl::StatusOr RunExpression( for (auto value : values) { auto& expr0 = create_list.mutable_elements().emplace_back().mutable_expr(); expr0.mutable_const_expr().set_int64_value(value); - path.push_back(ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::interop_internal::CreateIntValue(value)))); + path.push_back(ExpressionStep::MakeConstant( + cel::interop_internal::CreateIntValue(value))); } CEL_ASSIGN_OR_RETURN(auto step, CreateCreateListStep(create_list)); diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 0aa6000a4..7b3cceb3c 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -15,6 +15,9 @@ #include "eval/eval/evaluator_core.h" #include +#include +#include +#include #include #include "absl/base/nullability.h" @@ -25,6 +28,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "common/value.h" +#include "common/value_kind.h" #include "runtime/activation_interface.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -102,6 +106,41 @@ class EvaluationStatus final { } // namespace +void ExpressionStep::Evaluate(ExecutionFrame* context) const { + switch (header_.kind) { + case ExpressionStepKind::kGenericLogic: { + EvaluationStatus s(u_.logic->Evaluate(context)); + if (!s.ok()) { + context->Abort(std::move(s).Consume()); + } + break; + } + case ExpressionStepKind::kIntConstant: + context->value_stack().Push(cel::IntValue(u_.int_val)); + break; + case ExpressionStepKind::kBoolConstant: + context->value_stack().Push(cel::BoolValue(u_.bool_val)); + break; + case ExpressionStepKind::kDoubleConstant: + context->value_stack().Push(cel::DoubleValue(u_.double_val)); + break; + case ExpressionStepKind::kNullConstant: + context->value_stack().Push(cel::NullValue()); + break; + case ExpressionStepKind::kUintConstant: + context->value_stack().Push(cel::UintValue(u_.uint_val)); + break; + case ExpressionStepKind::kOtherConstant: + context->value_stack().Push(*u_.other_val); + break; + case ExpressionStepKind::kMovedFrom: + default: + context->Abort( + absl::InternalError("ExpressionStep::Evaluate called on moved-from " + "object")); + } +} + absl::StatusOr ExecutionFrame::Evaluate( EvaluationListener& listener) { const size_t initial_stack_size = value_stack().size(); @@ -109,18 +148,13 @@ absl::StatusOr ExecutionFrame::Evaluate( if (!listener) { for (const ExpressionStep* expr = Next(); ABSL_PREDICT_TRUE(expr != nullptr); expr = Next()) { - if (EvaluationStatus status(expr->Evaluate(this)); !status.ok()) { - return std::move(status).Consume(); - } + expr->Evaluate(this); } } else { for (const ExpressionStep* expr = Next(); ABSL_PREDICT_TRUE(expr != nullptr); expr = Next()) { - if (EvaluationStatus status(expr->Evaluate(this)); !status.ok()) { - return std::move(status).Consume(); - } - - if (pc_ == 0 || !expr->comes_from_ast()) { + expr->Evaluate(this); + if (pc_ == 0 || !expr->comes_from_ast() || !abort_status().ok()) { // Skip if we just started a Call or if the step doesn't map to an // AST id. continue; @@ -140,6 +174,10 @@ absl::StatusOr ExecutionFrame::Evaluate( } } + if (!abort_status().ok()) { + return std::move(abort_status()); + } + const size_t final_stack_size = value_stack().size(); if (ABSL_PREDICT_FALSE(final_stack_size != initial_stack_size + 1 || final_stack_size == 0)) { @@ -174,4 +212,80 @@ absl::StatusOr FlatExpression::EvaluateWithCallback( return frame.Evaluate(frame.callback()); } +ExpressionStep ExpressionStep::MakeConstant(const cel::Value& value, + int64_t id) { + if (id < 0 || id > std::numeric_limits::max()) { + id = -1; + } + int32_t id32 = static_cast(id); + switch (value.kind()) { + case cel::ValueKind::kBool: { + ExpressionStep step(ExpressionStepKind::kBoolConstant, id32); + step.u_.bool_val = value.GetBool().NativeValue(); + return step; + } + case cel::ValueKind::kInt: { + ExpressionStep step(ExpressionStepKind::kIntConstant, id32); + step.u_.int_val = value.GetInt().NativeValue(); + return step; + } + case cel::ValueKind::kUint: { + ExpressionStep step(ExpressionStepKind::kUintConstant, id32); + step.u_.uint_val = value.GetUint().NativeValue(); + return step; + } + case cel::ValueKind::kDouble: { + ExpressionStep step(ExpressionStepKind::kDoubleConstant, id32); + step.u_.double_val = value.GetDouble().NativeValue(); + return step; + } + case cel::ValueKind::kNull: + return ExpressionStep(ExpressionStepKind::kNullConstant, id32); + default: { + ExpressionStep step(ExpressionStepKind::kOtherConstant, id32); + step.u_.other_val = new cel::Value(value); + return step; + } + } +} + +bool GetIfConstant(const ExpressionStep& step, cel::Value& out) { + switch (step.header_.kind) { + case ExpressionStepKind::kIntConstant: + out = cel::IntValue(step.u_.int_val); + return true; + case ExpressionStepKind::kBoolConstant: + out = cel::BoolValue(step.u_.bool_val); + return true; + case ExpressionStepKind::kDoubleConstant: + out = cel::DoubleValue(step.u_.double_val); + return true; + case ExpressionStepKind::kNullConstant: + out = cel::NullValue(); + return true; + case ExpressionStepKind::kUintConstant: + out = cel::UintValue(step.u_.uint_val); + return true; + case ExpressionStepKind::kOtherConstant: + out = *step.u_.other_val; + return true; + default: + return false; + } +} + +bool IsConstant(const ExpressionStep& step) { + switch (step.header_.kind) { + case ExpressionStepKind::kIntConstant: + case ExpressionStepKind::kBoolConstant: + case ExpressionStepKind::kDoubleConstant: + case ExpressionStepKind::kNullConstant: + case ExpressionStepKind::kUintConstant: + case ExpressionStepKind::kOtherConstant: + return true; + default: + return false; + } +} + } // namespace google::api::expr::runtime diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index f25be8448..4b2fb4ad3 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -62,6 +62,13 @@ class ExpressionStepLogic; enum class ExpressionStepKind : uint16_t { kMovedFrom = 0, kGenericLogic = 1, + kIntConstant = 2, + kBoolConstant = 3, + kDoubleConstant = 4, + kNullConstant = 5, + kUintConstant = 6, + // Any constant that can't be inlined. + kOtherConstant = 7, }; class ExpressionStep { @@ -72,6 +79,8 @@ class ExpressionStep { ExpressionStep(ExpressionStep&&); ExpressionStep& operator=(ExpressionStep&&); + ~ExpressionStep(); + // Returns corresponding expression object ID. // Requires that the input expression has IDs assigned to sub-expressions, // e.g. via a checker. The default value 0 is returned if there is no @@ -85,12 +94,7 @@ class ExpressionStep { // Returns if the execution step comes from AST. bool comes_from_ast() const { return header_.id >= 0; } - absl::Status Evaluate(ExecutionFrame* context) const; - - template - T Get() const; - template - bool Is() const; + void Evaluate(ExecutionFrame* context) const; const ExpressionStepLogic* GetGenericStep() const; bool IsGenericStep() const; @@ -104,6 +108,8 @@ class ExpressionStep { std::move(logic)); } + static ExpressionStep MakeConstant(const cel::Value& value, int64_t id = -1); + private: struct Header { ExpressionStepKind kind; @@ -111,18 +117,51 @@ class ExpressionStep { int32_t id; }; - friend void swap(ExpressionStep& lhs, ExpressionStep& rhs); - - ExpressionStep() = default; - + ExpressionStep() : header_{ExpressionStepKind::kMovedFrom, 0, -1} {} + ExpressionStep(ExpressionStepKind kind, int32_t id) : header_{kind, 0, id} { + header_ = {kind, 0, id}; + } ExpressionStep(ExpressionStepKind kind, int32_t id, std::unique_ptr logic) - : header_{kind, 0, id}, logic_(std::move(logic)) {} + : ExpressionStep(kind, id) { + u_.logic = logic.release(); + } + + friend void swap(ExpressionStep& lhs, ExpressionStep& rhs) { + using std::swap; + swap(lhs.header_, rhs.header_); + swap(lhs.u_, rhs.u_); + } + + friend bool GetIfConstant(const ExpressionStep& step, cel::Value& out); + friend bool IsConstant(const ExpressionStep& step); Header header_; - std::unique_ptr logic_; + // Note: ptr members are 'owned' by the step. + // ~ExpressionStep() is responsible for freeing. + union Data { + std::nullptr_t empty; + ExpressionStepLogic* logic; + int64_t int_val; + uint64_t uint_val; + double double_val; + bool bool_val; + cel::Value* other_val; + + Data() : empty(nullptr) {} + ~Data() {} + } u_; }; +#ifndef _MSC_VER +// Keep the core instruction size small to get better memory locality for the +// main program. +// +// MSVC does not support some of the bit-field packing used here so it will be +// larger. +static_assert(sizeof(ExpressionStep) == 16); +#endif + // Class Expression represents single execution step. class ExpressionStepLogic { public: @@ -346,6 +385,8 @@ class ExecutionFrameBase { return absl::OkStatus(); } + absl::Status& abort_status() { return abort_status_; } + protected: const cel::ActivationInterface* absl_nonnull activation_; EvaluationListener callback_; @@ -359,6 +400,7 @@ class ExecutionFrameBase { ComprehensionSlots* absl_nonnull slots_; const int max_iterations_; int iterations_; + absl::Status abort_status_; const bool attribute_tracking_enabled_; const bool missing_attribute_errors_enabled_; const bool unknown_processing_enabled_; @@ -387,7 +429,7 @@ class ExecutionFrame : public ExecutionFrameBase { execution_path_(flat), value_stack_(&state.value_stack()), iterator_stack_(&state.iterator_stack()), - subexpressions_() {} + subexpressions_(&execution_path_, 1) {} ExecutionFrame( absl::Span subexpressions, @@ -487,6 +529,15 @@ class ExecutionFrame : public ExecutionFrameBase { return *activation_; } + void Abort(absl::Status status) { + ABSL_DCHECK(!subexpressions_.empty()); + ABSL_DCHECK(!status.ok()); + abort_status_.Update(std::move(status)); + call_stack_.clear(); + execution_path_ = subexpressions_[0]; + pc_ = execution_path_.size(); + } + private: struct SubFrame { size_t return_pc; @@ -580,36 +631,43 @@ class FlatExpression { absl_nullable std::shared_ptr arena_; }; -// Implementation details. +// Helper functions for checking ExpressionStep kinds. Used for program +// optimization. -inline absl::Status ExpressionStep::Evaluate(ExecutionFrame* context) const { - ABSL_DCHECK_EQ(header_.kind, ExpressionStepKind::kGenericLogic); - return logic_->Evaluate(context); -} +// Checks if the step is a constant and if so, writes the value into `out`. +// Returns true if the step is a constant, false otherwise. +bool GetIfConstant(const ExpressionStep& step, cel::Value& out); + +// Checks if the step is a constant. +bool IsConstant(const ExpressionStep& step); + +// Implementation details. -template -inline T ExpressionStep::Get() const { - if constexpr (std::is_same_v) { - return GetGenericStep(); - } else { - static_assert(sizeof(T) == 0, "unsupported type"); +inline ExpressionStep::~ExpressionStep() { + switch (header_.kind) { + case ExpressionStepKind::kGenericLogic: + delete u_.logic; + break; + case ExpressionStepKind::kOtherConstant: + delete u_.other_val; + break; + case ExpressionStepKind::kMovedFrom: + case ExpressionStepKind::kIntConstant: + case ExpressionStepKind::kBoolConstant: + case ExpressionStepKind::kDoubleConstant: + case ExpressionStepKind::kNullConstant: + case ExpressionStepKind::kUintConstant: + break; + default: + ABSL_UNREACHABLE(); } - ABSL_UNREACHABLE(); + header_.kind = ExpressionStepKind::kMovedFrom; + u_.empty = nullptr; } inline const ExpressionStepLogic* ExpressionStep::GetGenericStep() const { ABSL_DCHECK_EQ(header_.kind, ExpressionStepKind::kGenericLogic); - return logic_.get(); -} - -template -inline bool ExpressionStep::Is() const { - if constexpr (std::is_same_v) { - return header_.kind == ExpressionStepKind::kGenericLogic; - } else { - static_assert(sizeof(T) == 0, "unsupported type"); - } - ABSL_UNREACHABLE(); + return u_.logic; } inline bool ExpressionStep::IsGenericStep() const { @@ -619,22 +677,17 @@ inline bool ExpressionStep::IsGenericStep() const { inline ExpressionStep::ExpressionStep(ExpressionStep&& other) : ExpressionStep() { using std::swap; - swap(*this, other); + swap(other, *this); } inline ExpressionStep& ExpressionStep::operator=(ExpressionStep&& other) { using std::swap; - swap(*this, other); + ExpressionStep tmp; + swap(*this, tmp); + swap(other, *this); return *this; } -inline void swap(ExpressionStep& lhs, ExpressionStep& rhs) { - using std::swap; - - swap(lhs.header_, rhs.header_); - swap(lhs.logic_, rhs.logic_); -} - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_EVALUATOR_CORE_H_ diff --git a/eval/eval/evaluator_core_test.cc b/eval/eval/evaluator_core_test.cc index 873bc7365..91d03b749 100644 --- a/eval/eval/evaluator_core_test.cc +++ b/eval/eval/evaluator_core_test.cc @@ -121,6 +121,54 @@ TEST(EvaluatorCoreTest, SimpleEvaluatorTest) { EXPECT_THAT(value.Int64OrDie(), Eq(2)); } +TEST(EvaluatorCoreTest, MakeConstant) { + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + google::protobuf::Arena arena; + cel::Activation activation; + cel::RuntimeOptions options; + + auto evaluate_constant = + [&](const cel::Value& value) -> absl::StatusOr { + ExecutionPath path; + path.push_back(ExpressionStep::MakeConstant(value)); + FlatExpression expr(std::move(path), 0, type_provider, options); + auto state = expr.MakeEvaluatorState( + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + return expr.EvaluateWithCallback(activation, nullptr, nullptr, state); + }; + + ASSERT_OK_AND_ASSIGN(auto bool_val, evaluate_constant(cel::BoolValue(true))); + EXPECT_TRUE(bool_val.IsBool()); + EXPECT_TRUE(bool_val.GetBool().NativeValue()); + + ASSERT_OK_AND_ASSIGN(auto int_val, evaluate_constant(cel::IntValue(42))); + EXPECT_TRUE(int_val.IsInt()); + EXPECT_EQ(int_val.GetInt().NativeValue(), 42); + + ASSERT_OK_AND_ASSIGN(auto uint_val, evaluate_constant(cel::UintValue(100))); + EXPECT_TRUE(uint_val.IsUint()); + EXPECT_EQ(uint_val.GetUint().NativeValue(), 100); + + ASSERT_OK_AND_ASSIGN(auto double_val, + evaluate_constant(cel::DoubleValue(3.14))); + EXPECT_TRUE(double_val.IsDouble()); + EXPECT_DOUBLE_EQ(double_val.GetDouble().NativeValue(), 3.14); + + ASSERT_OK_AND_ASSIGN(auto null_val, evaluate_constant(cel::NullValue())); + EXPECT_TRUE(null_val.IsNull()); + + ASSERT_OK_AND_ASSIGN(auto str_val, + evaluate_constant(cel::StringValue::Literal("hello"))); + EXPECT_TRUE(str_val.IsString()); + EXPECT_EQ(str_val.GetString().ToString(), "hello"); + + auto step_with_id = ExpressionStep::MakeConstant(cel::IntValue(1), 123); + EXPECT_EQ(step_with_id.id(), 123); + EXPECT_TRUE(step_with_id.comes_from_ast()); +} + class MockTraceCallback { public: MOCK_METHOD(void, Call, diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index 8adbda76c..a436a3281 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -8,6 +8,7 @@ #include #include "absl/status/status.h" +#include "absl/status/status_macros.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -233,13 +234,13 @@ absl::StatusOr MakeTestFunctionStep( call.function(), call.has_target(), argument_matcher); int id = GetExprId(); if (!lazy_overloads.empty()) { - CEL_ASSIGN_OR_RETURN(auto logic, - CreateFunctionStep(call, id, lazy_overloads)); + ABSL_ASSIGN_OR_RETURN(auto logic, + CreateFunctionStep(call, id, lazy_overloads)); return ExpressionStep::MakeGenericStep(std::move(logic), id); } auto overloads = registry.FindStaticOverloads( call.function(), call.has_target(), argument_matcher); - CEL_ASSIGN_OR_RETURN(auto logic, CreateFunctionStep(call, id, overloads)); + ABSL_ASSIGN_OR_RETURN(auto logic, CreateFunctionStep(call, id, overloads)); return ExpressionStep::MakeGenericStep(std::move(logic), id); } @@ -540,11 +541,11 @@ TEST_P(FunctionStepTest, LazyFunctionOverloadingTest) { lt_call.mutable_args().emplace_back(); lt_call.set_function("_<_"); - auto step0 = ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::interop_internal::CreateIntValue(20))); + auto step0 = + ExpressionStep::MakeConstant(cel::interop_internal::CreateIntValue(20)); ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); - auto step2 = ExpressionStep::MakeGenericStep( - CreateConstValueStep(cel::interop_internal::CreateDoubleValue(21.9))); + auto step2 = ExpressionStep::MakeConstant( + cel::interop_internal::CreateDoubleValue(21.9)); ASSERT_OK_AND_ASSIGN(auto step3, MakeTestFunctionStep(call2, registry)); ASSERT_OK_AND_ASSIGN(auto step4, MakeTestFunctionStep(lt_call, registry)); diff --git a/eval/eval/lazy_init_step_test.cc b/eval/eval/lazy_init_step_test.cc index 00490dbe1..20105dfa0 100644 --- a/eval/eval/lazy_init_step_test.cc +++ b/eval/eval/lazy_init_step_test.cc @@ -19,7 +19,6 @@ #include "base/type_provider.h" #include "common/value.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/evaluator_core.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" @@ -65,8 +64,7 @@ TEST_F(LazyInitStepTest, CreateCheckInitStepDoesInit) { path.push_back(ExpressionStep::MakeGenericStep(CreateLazyInitStep( /*slot_index=*/0, /*subexpression_index=*/1))); - subpath.push_back( - ExpressionStep::MakeGenericStep(CreateConstValueStep(cel::IntValue(42)))); + subpath.push_back(ExpressionStep::MakeConstant(cel::IntValue(42))); std::vector expression_table{path, subpath}; @@ -87,8 +85,7 @@ TEST_F(LazyInitStepTest, CreateCheckInitStepSkipInit) { path.push_back(ExpressionStep::MakeGenericStep(CreateLazyInitStep( /*slot_index=*/0, /*subexpression_index=*/-1))); - subpath.push_back( - ExpressionStep::MakeGenericStep(CreateConstValueStep(cel::IntValue(42)))); + subpath.push_back(ExpressionStep::MakeConstant(cel::IntValue(42))); std::vector expression_table{path, subpath}; From 8c77c8fd80e923c489f1cc40cfcde9e4738ec083 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 29 Sep 2026 12:30:51 -0700 Subject: [PATCH 246/350] Add builtin slot and subexpression lazy init instructions. PiperOrigin-RevId: 990464095 --- eval/compiler/BUILD | 1 - eval/compiler/flat_expr_builder.cc | 21 +++- eval/compiler/flat_expr_builder_extensions.h | 5 + eval/eval/BUILD | 29 ++--- eval/eval/comprehension_slots.h | 7 +- eval/eval/direct_expression_step.cc | 16 +-- eval/eval/direct_expression_step.h | 22 +--- eval/eval/evaluator_core.cc | 20 +++ eval/eval/evaluator_core.h | 76 +++++++++++- eval/eval/lazy_init_step.cc | 124 ++++--------------- eval/eval/lazy_init_step.h | 29 ++--- eval/eval/lazy_init_step_test.cc | 40 +++--- 12 files changed, 185 insertions(+), 205 deletions(-) diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 6916c96a5..76a20f5a8 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -123,7 +123,6 @@ cc_library( "//eval/eval:function_step", "//eval/eval:ident_step", "//eval/eval:jump_step", - "//eval/eval:lazy_init_step", "//eval/eval:logic_step", "//eval/eval:optional_or_step", "//eval/eval:select_step", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 7cb94fe45..ef36d6e94 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -855,8 +855,8 @@ class FlatExprVisitor : public cel::AstVisitor { program.depth + 1); } else { // Off by one since mainline expression will be index 0. - AddStep(CreateLazyInitStep(slot.slot, slot.subexpression + 1), - expr.id()); + AddStep(ExpressionStep::MakeLazyInitStep( + slot.slot, slot.subexpression + 1, expr.id())); } return; } else if (slot.slot >= 0) { @@ -2134,7 +2134,8 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleBlock( // Otherwise, iterative plan. if (block.slot_count > 0) { - AddStep(CreateClearSlotsStep(block.index, block.slot_count), expr.id()); + AddStep(ExpressionStep::MakeClearSlotsStep(block.index, block.slot_count, + expr.id())); } return CallHandlerResult::kIntercepted; @@ -2518,7 +2519,7 @@ void ComprehensionVisitor::PostVisitArgTrivial(cel::ComprehensionArg arg_num, } case cel::ACCU_INIT: { if (!accu_init_extracted_) { - visitor_->AddStep(CreateAssignSlotAndPopStep(accu_slot_)); + visitor_->AddStep(ExpressionStep::MakeAssignSlotAndPopStep(accu_slot_)); } break; } @@ -2529,7 +2530,8 @@ void ComprehensionVisitor::PostVisitArgTrivial(cel::ComprehensionArg arg_num, break; } case cel::RESULT: { - visitor_->AddStep(CreateClearSlotStep(accu_slot_), expr->id()); + visitor_->AddStep( + ExpressionStep::MakeClearSlotStep(accu_slot_, expr->id())); break; } } @@ -2657,6 +2659,15 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( return visitor.progress_status(); } + if (visitor.slot_count() > std::numeric_limits::max() || + program_builder.ExtractedSubexpressionCount() > + std::numeric_limits::max()) { + // Impractical to trigger (we'd run out of memory first), but assuming this + // allows us to pack references to slots and subexpressions. + return absl::InternalError( + "Expression too large to be executed, exceeds uint32_t limits."); + } + if (issues != nullptr) { (*issues) = issue_collector.ExtractIssues(); } diff --git a/eval/compiler/flat_expr_builder_extensions.h b/eval/compiler/flat_expr_builder_extensions.h index 0112356e4..2ce70a166 100644 --- a/eval/compiler/flat_expr_builder_extensions.h +++ b/eval/compiler/flat_expr_builder_extensions.h @@ -233,6 +233,11 @@ class ProgramBuilder { // programs table to starting state. std::vector FlattenSubexpressions(); + // Return the current number of subexpressions in the program builder. + size_t ExtractedSubexpressionCount() const { + return extracted_subexpressions_.size(); + } + // Returns the current subexpression where steps and new subexpressions are // added. // diff --git a/eval/eval/BUILD b/eval/eval/BUILD index de692b8bf..41aca62b9 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -36,19 +36,24 @@ cc_library( name = "evaluator_core", srcs = [ "evaluator_core.cc", + "lazy_init_step.cc", ], hdrs = [ "evaluator_core.h", + "lazy_init_step.h", ], deps = [ + ":attribute_trail", ":attribute_utility", ":comprehension_slots", + ":direct_expression_step", ":evaluator_stack", ":iterator_stack", "//base:data", "//common:native_type", "//common:value", "//common:value_kind", + "//internal:status_macros", "//runtime", "//runtime:activation_interface", "//runtime:runtime_options", @@ -61,6 +66,7 @@ cc_library( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:span", + "@com_google_cel_spec//proto/cel/expr:value_cc_proto", "@com_google_protobuf//:protobuf", ], ) @@ -1164,31 +1170,12 @@ cc_test( ], ) -cc_library( - name = "lazy_init_step", - srcs = ["lazy_init_step.cc"], - hdrs = ["lazy_init_step.h"], - deps = [ - ":attribute_trail", - ":comprehension_slots", - ":direct_expression_step", - ":evaluator_core", - ":expression_step_base", - "//common:value", - "//internal:status_macros", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status", - "@com_google_cel_spec//proto/cel/expr:value_cc_proto", - ], -) - cc_test( name = "lazy_init_step_test", srcs = ["lazy_init_step_test.cc"], deps = [ - ":const_value_step", + ":comprehension_slots", ":evaluator_core", - ":lazy_init_step", "//base:data", "//common:value", "//internal:testing", @@ -1207,10 +1194,8 @@ cc_library( hdrs = ["direct_expression_step.h"], deps = [ ":attribute_trail", - ":evaluator_core", "//common:native_type", "//common:value", - "//internal:status_macros", "@com_google_absl//absl/status", "@com_google_absl//absl/types:optional", ], diff --git a/eval/eval/comprehension_slots.h b/eval/eval/comprehension_slots.h index 795cca7f7..5a7ce21cc 100644 --- a/eval/eval/comprehension_slots.h +++ b/eval/eval/comprehension_slots.h @@ -16,6 +16,8 @@ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPREHENSION_SLOTS_H_ #include +#include +#include #include #include "absl/base/attributes.h" @@ -110,7 +112,10 @@ class ComprehensionSlots final { return *instance; } - explicit ComprehensionSlots(size_t size) : slots_(size) {} + explicit ComprehensionSlots(size_t size) : slots_(size) { + ABSL_DCHECK_LT(size, + static_cast(std::numeric_limits::max())); + } ComprehensionSlots(const ComprehensionSlots&) = delete; ComprehensionSlots& operator=(const ComprehensionSlots&) = delete; diff --git a/eval/eval/direct_expression_step.cc b/eval/eval/direct_expression_step.cc index 2d7fc6fc0..36b5ed1f5 100644 --- a/eval/eval/direct_expression_step.cc +++ b/eval/eval/direct_expression_step.cc @@ -13,22 +13,8 @@ // limitations under the License. #include "eval/eval/direct_expression_step.h" -#include - -#include "absl/status/status.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/evaluator_core.h" -#include "internal/status_macros.h" - namespace google::api::expr::runtime { -absl::Status WrappedDirectStep::Evaluate(ExecutionFrame* frame) const { - cel::Value result; - AttributeTrail attribute_trail; - CEL_RETURN_IF_ERROR(impl_->Evaluate(*frame, result, attribute_trail)); - frame->value_stack().Push(std::move(result), std::move(attribute_trail)); - return absl::OkStatus(); -} +// Intentionally empty. } // namespace google::api::expr::runtime diff --git a/eval/eval/direct_expression_step.h b/eval/eval/direct_expression_step.h index ac783d2d7..547adad41 100644 --- a/eval/eval/direct_expression_step.h +++ b/eval/eval/direct_expression_step.h @@ -24,10 +24,11 @@ #include "common/native_type.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { +class ExecutionFrameBase; + // Represents a directly evaluated CEL expression. // // Subexpressions assign to values on the C++ program stack and call their @@ -76,25 +77,6 @@ class DirectExpressionStep { int64_t expr_id_; }; -// Wrapper for direct steps to work with the stack machine impl. -class WrappedDirectStep : public ExpressionStepLogic { - public: - explicit WrappedDirectStep(std::unique_ptr impl, - int64_t expr_id = -1) - : impl_(std::move(impl)) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - const DirectExpressionStep* wrapped() const { return impl_.get(); } - - private: - std::unique_ptr impl_; -}; - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_DIRECT_EXPRESSION_STEP_H_ diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 7b3cceb3c..ba275ec80 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -29,6 +29,9 @@ #include "absl/strings/str_cat.h" #include "common/value.h" #include "common/value_kind.h" +#include "eval/eval/attribute_trail.h" +#include "eval/eval/lazy_init_step.h" +#include "internal/status_macros.h" #include "runtime/activation_interface.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -133,6 +136,15 @@ void ExpressionStep::Evaluate(ExecutionFrame* context) const { case ExpressionStepKind::kOtherConstant: context->value_stack().Push(*u_.other_val); break; + case ExpressionStepKind::kLazyInit: + EvaluateLazyInitStep(u_.lazy_init, *context); + break; + case ExpressionStepKind::kAssignSlotAndPop: + EvaluateAssignSlotAndPop(u_.slot_index, *context); + break; + case ExpressionStepKind::kClearSlots: + EvaluateClearSlotStep(u_.clear_slots, *context); + break; case ExpressionStepKind::kMovedFrom: default: context->Abort( @@ -288,4 +300,12 @@ bool IsConstant(const ExpressionStep& step) { } } +absl::Status WrappedDirectStep::Evaluate(ExecutionFrame* frame) const { + cel::Value result; + AttributeTrail attribute_trail; + CEL_RETURN_IF_ERROR(impl_->Evaluate(*frame, result, attribute_trail)); + frame->value_stack().Push(std::move(result), std::move(attribute_trail)); + return absl::OkStatus(); +} + } // namespace google::api::expr::runtime diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 4b2fb4ad3..89a98ca2c 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -36,8 +35,10 @@ #include "common/value.h" #include "eval/eval/attribute_utility.h" #include "eval/eval/comprehension_slots.h" +#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_stack.h" #include "eval/eval/iterator_stack.h" +#include "eval/eval/lazy_init_step.h" #include "runtime/activation_interface.h" #include "runtime/internal/activation_attribute_matcher_access.h" #include "runtime/runtime.h" @@ -69,6 +70,9 @@ enum class ExpressionStepKind : uint16_t { kUintConstant = 6, // Any constant that can't be inlined. kOtherConstant = 7, + kLazyInit = 8, + kAssignSlotAndPop = 9, + kClearSlots = 10, }; class ExpressionStep { @@ -101,15 +105,44 @@ class ExpressionStep { static ExpressionStep MakeGenericStep( std::unique_ptr logic, int64_t id = -1) { - if (id < 0 || id > std::numeric_limits::max()) { - id = -1; - } - return ExpressionStep(ExpressionStepKind::kGenericLogic, id, - std::move(logic)); + ExpressionStep step(ExpressionStepKind::kGenericLogic, id); + step.u_.logic = logic.release(); + return step; } static ExpressionStep MakeConstant(const cel::Value& value, int64_t id = -1); + static ExpressionStep MakeLazyInitStep(size_t slot_index, + size_t subexpression_index, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kLazyInit, id); + ABSL_DCHECK_LE(slot_index, std::numeric_limits::max()); + ABSL_DCHECK_LE(subexpression_index, std::numeric_limits::max()); + step.u_.lazy_init = LazyInitStepInfo{slot_index, subexpression_index}; + return step; + } + + static ExpressionStep MakeAssignSlotAndPopStep(size_t slot_index, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kAssignSlotAndPop, id); + ABSL_DCHECK_LE(slot_index, std::numeric_limits::max()); + step.u_.slot_index = slot_index; + return step; + } + + static ExpressionStep MakeClearSlotStep(size_t slot_index, int64_t id = -1) { + return MakeClearSlotsStep(slot_index, 1, id); + } + + static ExpressionStep MakeClearSlotsStep(size_t slot_index, size_t slot_count, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kClearSlots, id); + ABSL_DCHECK_LE(slot_index, std::numeric_limits::max()); + ABSL_DCHECK_LE(slot_count, std::numeric_limits::max()); + step.u_.clear_slots = ClearSlotStepInfo{slot_index, slot_count}; + return step; + } + private: struct Header { ExpressionStepKind kind; @@ -121,6 +154,12 @@ class ExpressionStep { ExpressionStep(ExpressionStepKind kind, int32_t id) : header_{kind, 0, id} { header_ = {kind, 0, id}; } + ExpressionStep(ExpressionStepKind kind, int64_t id) : ExpressionStep() { + if (id < 0 || id > std::numeric_limits::max()) { + id = -1; + } + header_ = {kind, 0, static_cast(id)}; + } ExpressionStep(ExpressionStepKind kind, int32_t id, std::unique_ptr logic) : ExpressionStep(kind, id) { @@ -147,6 +186,9 @@ class ExpressionStep { double double_val; bool bool_val; cel::Value* other_val; + LazyInitStepInfo lazy_init; + size_t slot_index; + ClearSlotStepInfo clear_slots; Data() : empty(nullptr) {} ~Data() {} @@ -184,6 +226,25 @@ class ExpressionStepLogic { } }; +// Wrapper for direct steps to work with the stack machine impl. +class WrappedDirectStep : public ExpressionStepLogic { + public: + explicit WrappedDirectStep(std::unique_ptr impl, + int64_t expr_id = -1) + : impl_(std::move(impl)) {} + + absl::Status Evaluate(ExecutionFrame* frame) const override; + + cel::NativeTypeId GetNativeTypeId() const override { + return cel::NativeTypeId::For(); + } + + const DirectExpressionStep* wrapped() const { return impl_.get(); } + + private: + std::unique_ptr impl_; +}; + using ExecutionPath = std::vector; using ExecutionPathView = absl::Span; @@ -657,6 +718,9 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kDoubleConstant: case ExpressionStepKind::kNullConstant: case ExpressionStepKind::kUintConstant: + case ExpressionStepKind::kLazyInit: + case ExpressionStepKind::kAssignSlotAndPop: + case ExpressionStepKind::kClearSlots: break; default: ABSL_UNREACHABLE(); diff --git a/eval/eval/lazy_init_step.cc b/eval/eval/lazy_init_step.cc index 7d5fa538a..cdb1f9666 100644 --- a/eval/eval/lazy_init_step.cc +++ b/eval/eval/lazy_init_step.cc @@ -27,7 +27,6 @@ #include "eval/eval/comprehension_slots.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" #include "internal/status_macros.h" namespace google::api::expr::runtime { @@ -36,28 +35,6 @@ namespace { using ::cel::Value; -class LazyInitStep final : public ExpressionStepBase { - public: - LazyInitStep(size_t slot_index, size_t subexpression_index) - : ExpressionStepBase(), - slot_index_(slot_index), - subexpression_index_(subexpression_index) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - ComprehensionSlot* slot = frame->comprehension_slots().Get(slot_index_); - if (slot->Has()) { - frame->value_stack().Push(slot->value(), slot->attribute()); - } else { - frame->Call(slot_index_, subexpression_index_); - } - return absl::OkStatus(); - } - - private: - const size_t slot_index_; - const size_t subexpression_index_; -}; - class DirectLazyInitStep final : public DirectExpressionStep { public: DirectLazyInitStep(size_t slot_index, @@ -106,61 +83,6 @@ class BindStep : public DirectExpressionStep { std::unique_ptr subexpression_; }; -class AssignSlotAndPopStepStep final : public ExpressionStepBase { - public: - explicit AssignSlotAndPopStepStep(size_t slot_index) - : ExpressionStepBase(/*expr_id=*/-1, /*comes_from_ast=*/false), - slot_index_(slot_index) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError("Stack underflow assigning lazy value"); - } - - frame->comprehension_slots().Set(slot_index_, frame->value_stack().Peek(), - frame->value_stack().PeekAttribute()); - frame->value_stack().Pop(1); - - return absl::OkStatus(); - } - - private: - const size_t slot_index_; -}; - -class ClearSlotStep : public ExpressionStepBase { - public: - explicit ClearSlotStep(size_t slot_index) - : ExpressionStepBase(), slot_index_(slot_index) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - frame->comprehension_slots().ClearSlot(slot_index_); - return absl::OkStatus(); - } - - private: - size_t slot_index_; -}; - -class ClearSlotsStep final : public ExpressionStepBase { - public: - ClearSlotsStep(size_t slot_index, size_t slot_count) - : ExpressionStepBase(), - slot_index_(slot_index), - slot_count_(slot_count) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - for (size_t i = 0; i < slot_count_; ++i) { - frame->comprehension_slots().ClearSlot(slot_index_ + i); - } - return absl::OkStatus(); - } - - private: - const size_t slot_index_; - const size_t slot_count_; -}; - class BlockStep : public DirectExpressionStep { public: BlockStep(size_t slot_index, size_t slot_count, @@ -190,6 +112,32 @@ class BlockStep : public DirectExpressionStep { } // namespace +void EvaluateLazyInitStep(const LazyInitStepInfo& step, ExecutionFrame& frame) { + ComprehensionSlot* slot = frame.comprehension_slots().Get(step.slot_index); + if (slot->Has()) { + frame.value_stack().Push(slot->value(), slot->attribute()); + } else { + frame.Call(step.slot_index, step.subexpression_index); + } +} + +void EvaluateAssignSlotAndPop(size_t slot_index, ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(1)) { + frame.Abort(absl::InternalError("Stack underflow assigning lazy value")); + return; + } + ComprehensionSlot* slot = frame.comprehension_slots().Get(slot_index); + slot->Set(frame.value_stack().Peek(), frame.value_stack().PeekAttribute()); + frame.value_stack().Pop(1); +} + +void EvaluateClearSlotStep(const ClearSlotStepInfo& step, + ExecutionFrame& frame) { + for (size_t i = 0; i < step.slot_count; ++i) { + frame.comprehension_slots().ClearSlot(step.slot_index + i); + } +} + std::unique_ptr CreateDirectBindStep( size_t slot_index, std::unique_ptr expression, int64_t expr_id) { @@ -210,24 +158,4 @@ std::unique_ptr CreateDirectLazyInitStep( expr_id); } -std::unique_ptr CreateLazyInitStep( - size_t slot_index, size_t subexpression_index) { - return std::make_unique(slot_index, subexpression_index); -} - -std::unique_ptr CreateAssignSlotAndPopStep( - size_t slot_index) { - return std::make_unique(slot_index); -} - -std::unique_ptr CreateClearSlotStep(size_t slot_index) { - return std::make_unique(slot_index); -} - -std::unique_ptr CreateClearSlotsStep(size_t slot_index, - size_t slot_count) { - ABSL_DCHECK_GT(slot_count, 0); - return std::make_unique(slot_index, slot_count); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step.h b/eval/eval/lazy_init_step.h index e1ef59c93..c73c80ee1 100644 --- a/eval/eval/lazy_init_step.h +++ b/eval/eval/lazy_init_step.h @@ -43,10 +43,11 @@ #include "absl/base/nullability.h" #include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { +class ExecutionFrame; + // Creates a step representing a Bind expression. std::unique_ptr CreateDirectBindStep( size_t slot_index, std::unique_ptr expression, @@ -63,22 +64,22 @@ std::unique_ptr CreateDirectLazyInitStep( size_t slot_index, const DirectExpressionStep* absl_nonnull subexpression, int64_t expr_id); -// Creates a step representing accessing a lazily evaluated alias from -// a bind or block. -std::unique_ptr CreateLazyInitStep( - size_t slot_index, size_t subexpression_index); +struct LazyInitStepInfo { + size_t slot_index : 32; + size_t subexpression_index : 32; +}; + +void EvaluateLazyInitStep(const LazyInitStepInfo& step, ExecutionFrame& frame); -// Helper step to assign a slot value from the top of stack on initialization. -std::unique_ptr CreateAssignSlotAndPopStep( - size_t slot_index); +void EvaluateAssignSlotAndPop(size_t slot_index, ExecutionFrame& frame); -// Helper step to clear a slot. -// Slots may be reused in different contexts so need to be cleared after a -// context is done. -std::unique_ptr CreateClearSlotStep(size_t slot_index); +struct ClearSlotStepInfo { + size_t slot_index : 32; + size_t slot_count : 32 = 1; +}; -std::unique_ptr CreateClearSlotsStep(size_t slot_index, - size_t slot_count); +void EvaluateClearSlotStep(const ClearSlotStepInfo& step, + ExecutionFrame& frame); } // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step_test.cc b/eval/eval/lazy_init_step_test.cc index 20105dfa0..3b5282d21 100644 --- a/eval/eval/lazy_init_step_test.cc +++ b/eval/eval/lazy_init_step_test.cc @@ -19,6 +19,7 @@ #include "base/type_provider.h" #include "common/value.h" +#include "eval/eval/comprehension_slots.h" #include "eval/eval/evaluator_core.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" @@ -57,12 +58,12 @@ class LazyInitStepTest : public testing::Test { Activation activation_; }; -TEST_F(LazyInitStepTest, CreateCheckInitStepDoesInit) { +TEST_F(LazyInitStepTest, MakeLazyInitDoesInit) { ExecutionPath path; ExecutionPath subpath; - path.push_back(ExpressionStep::MakeGenericStep(CreateLazyInitStep( - /*slot_index=*/0, /*subexpression_index=*/1))); + path.push_back(ExpressionStep::MakeLazyInitStep( + /*slot_index=*/0, /*subexpression_index=*/1)); subpath.push_back(ExpressionStep::MakeConstant(cel::IntValue(42))); @@ -70,20 +71,17 @@ TEST_F(LazyInitStepTest, CreateCheckInitStepDoesInit) { ExecutionFrame frame(expression_table, activation_, runtime_options_, evaluator_state_); - ASSERT_OK_AND_ASSIGN(auto value, frame.Evaluate()); + ASSERT_OK_AND_ASSIGN(cel::Value value, frame.Evaluate()); EXPECT_TRUE(value->Is() && value.GetInt().NativeValue() == 42); } -TEST_F(LazyInitStepTest, CreateCheckInitStepSkipInit) { +TEST_F(LazyInitStepTest, MakeLazyInitSkipInit) { ExecutionPath path; ExecutionPath subpath; - // This is the expected usage, but in this test we are just depending on the - // fact that these don't change the stack and fit the program layout - // requirements. - path.push_back(ExpressionStep::MakeGenericStep(CreateLazyInitStep( - /*slot_index=*/0, /*subexpression_index=*/-1))); + path.push_back(ExpressionStep::MakeLazyInitStep( + /*slot_index=*/0, /*subexpression_index=*/2)); subpath.push_back(ExpressionStep::MakeConstant(cel::IntValue(42))); @@ -92,57 +90,53 @@ TEST_F(LazyInitStepTest, CreateCheckInitStepSkipInit) { ExecutionFrame frame(expression_table, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().Set(0, cel::IntValue(42)); - ASSERT_OK_AND_ASSIGN(auto value, frame.Evaluate()); + ASSERT_OK_AND_ASSIGN(cel::Value value, frame.Evaluate()); EXPECT_TRUE(value->Is() && value.GetInt().NativeValue() == 42); } -TEST_F(LazyInitStepTest, CreateAssignSlotAndPopStepBasic) { +TEST_F(LazyInitStepTest, MakeAssignSlotAndPopBasic) { ExecutionPath path; - path.push_back( - ExpressionStep::MakeGenericStep(CreateAssignSlotAndPopStep(0))); + path.push_back(ExpressionStep::MakeAssignSlotAndPopStep(0)); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().ClearSlot(0); frame.value_stack().Push(cel::IntValue(42)); - // This will error because no return value, step will still evaluate. frame.Evaluate().IgnoreError(); - auto* slot = frame.comprehension_slots().Get(0); + ComprehensionSlots::Slot* slot = frame.comprehension_slots().Get(0); ASSERT_TRUE(slot->Has()); EXPECT_TRUE(slot->value()->Is() && slot->value().GetInt().NativeValue() == 42); EXPECT_TRUE(frame.value_stack().empty()); } -TEST_F(LazyInitStepTest, CreateClearSlotStepBasic) { +TEST_F(LazyInitStepTest, MakeClearSlotBasic) { ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(CreateClearSlotStep(0))); + path.push_back(ExpressionStep::MakeClearSlotStep(0)); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().Set(0, cel::IntValue(42)); - // This will error because no return value, step will still evaluate. frame.Evaluate().IgnoreError(); - auto* slot = frame.comprehension_slots().Get(0); + ComprehensionSlot* slot = frame.comprehension_slots().Get(0); ASSERT_FALSE(slot->Has()); } -TEST_F(LazyInitStepTest, CreateClearSlotsStepBasic) { +TEST_F(LazyInitStepTest, MakeClearSlotsBasic) { ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(CreateClearSlotsStep(0, 2))); + path.push_back(ExpressionStep::MakeClearSlotsStep(0, 2)); ExecutionFrame frame(path, activation_, runtime_options_, evaluator_state_); frame.comprehension_slots().Set(0, cel::IntValue(42)); frame.comprehension_slots().Set(1, cel::IntValue(42)); - // This will error because no return value, step will still evaluate. frame.Evaluate().IgnoreError(); EXPECT_FALSE(frame.comprehension_slots().Get(0)->Has()); From 96346c7ad08ef86fd4f7abb53d68ea24be8a8ffd Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 29 Sep 2026 13:57:42 -0700 Subject: [PATCH 247/350] Skip unused attribute pattern lookups when constructing an ExecutionFrame. `ExecutionFrameBase` only calls `activation.GetUnknownAttributes()` / `GetMissingAttributes()` when unknown processing / missing attribute errors are enabled; otherwise it passes empty spans to `AttributeUtility`, which never consults them in that configuration. PiperOrigin-RevId: 990512440 --- eval/eval/evaluator_core.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 89a98ca2c..dea5f527f 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -321,8 +321,13 @@ class ExecutionFrameBase { message_factory_(message_factory), arena_(arena), embedder_context_(nullptr), - attribute_utility_(activation.GetUnknownAttributes(), - activation.GetMissingAttributes()), + attribute_utility_(options.unknown_processing != + cel::UnknownProcessingOptions::kDisabled + ? activation.GetUnknownAttributes() + : absl::Span(), + options.enable_missing_attribute_errors + ? activation.GetMissingAttributes() + : absl::Span()), slots_(&ComprehensionSlots::GetEmptyInstance()), max_iterations_(options.comprehension_max_iterations), iterations_(0), @@ -363,8 +368,13 @@ class ExecutionFrameBase { message_factory_(message_factory), arena_(arena), embedder_context_(embedder_context), - attribute_utility_(activation.GetUnknownAttributes(), - activation.GetMissingAttributes()), + attribute_utility_(options.unknown_processing != + cel::UnknownProcessingOptions::kDisabled + ? activation.GetUnknownAttributes() + : absl::Span(), + options.enable_missing_attribute_errors + ? activation.GetMissingAttributes() + : absl::Span()), slots_(&slots), max_iterations_(options.comprehension_max_iterations), iterations_(0), From 9602b673e77da7f92dfcefcd3cbd5ba466ee8bce Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Wed, 30 Sep 2026 09:27:41 -0700 Subject: [PATCH 248/350] Propagate unsafe field access through `ParsedMessageValue::Qualify`. Field selection on messages wrapped with the `*Unsafe` APIs now borrows sub-messages and strings instead of copying them into the evaluation arena, matching `ParsedMessageValue::GetField`. PiperOrigin-RevId: 991037181 --- common/values/parsed_message_value.cc | 35 ++++++-- common/values/parsed_message_value_test.cc | 96 ++++++++++++++++++++++ 2 files changed, 122 insertions(+), 9 deletions(-) diff --git a/common/values/parsed_message_value.cc b/common/values/parsed_message_value.cc index 62e3adc40..a7cc01e42 100644 --- a/common/values/parsed_message_value.cc +++ b/common/values/parsed_message_value.cc @@ -278,12 +278,13 @@ class ParsedMessageValueQualifyState final const google::protobuf::Message* absl_nonnull message, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena) + google::protobuf::Arena* absl_nonnull arena, bool unsafe) : ProtoQualifyState(message, message->GetDescriptor(), message->GetReflection()), descriptor_pool_(descriptor_pool), message_factory_(message_factory), - arena_(arena) {} + arena_(arena), + unsafe_(unsafe) {} absl::optional& result() { return result_; } @@ -294,11 +295,18 @@ class ParsedMessageValueQualifyState final void SetResultFromBool(bool value) override { result_ = BoolValue(value); } + // When `unsafe_` is set, the qualified message is externally managed (see + // `UnsafeParsedMessageValue()`), so borrow field values instead of copying + // them onto `arena_`. This matches `ParsedMessageValue::GetField()`. absl::Status SetResultFromField(const google::protobuf::Message* message, const google::protobuf::FieldDescriptor* field, ProtoWrapperTypeOptions unboxing_option, cel::MemoryManagerRef) override { - result_ = Value::WrapField(unboxing_option, message, field, + result_ = + unsafe_ + ? Value::WrapFieldUnsafe(unboxing_option, message, field, + descriptor_pool_, message_factory_, arena_) + : Value::WrapField(unboxing_option, message, field, descriptor_pool_, message_factory_, arena_); return absl::OkStatus(); } @@ -307,8 +315,12 @@ class ParsedMessageValueQualifyState final const google::protobuf::FieldDescriptor* field, int index, cel::MemoryManagerRef) override { - result_ = Value::WrapRepeatedField(index, message, field, descriptor_pool_, - message_factory_, arena_); + result_ = unsafe_ ? Value::WrapRepeatedFieldUnsafe(index, message, field, + descriptor_pool_, + message_factory_, arena_) + : Value::WrapRepeatedField(index, message, field, + descriptor_pool_, + message_factory_, arena_); return absl::OkStatus(); } @@ -316,14 +328,19 @@ class ParsedMessageValueQualifyState final const google::protobuf::FieldDescriptor* field, const google::protobuf::MapValueConstRef& value, cel::MemoryManagerRef) override { - result_ = Value::WrapMapFieldValue(value, message, field, descriptor_pool_, - message_factory_, arena_); + result_ = unsafe_ ? Value::WrapMapFieldValueUnsafe(value, message, field, + descriptor_pool_, + message_factory_, arena_) + : Value::WrapMapFieldValue(value, message, field, + descriptor_pool_, + message_factory_, arena_); return absl::OkStatus(); } const google::protobuf::DescriptorPool* absl_nonnull const descriptor_pool_; google::protobuf::MessageFactory* absl_nonnull const message_factory_; google::protobuf::Arena* absl_nonnull const arena_; + const bool unsafe_; absl::optional result_; }; @@ -345,8 +362,8 @@ absl::Status ParsedMessageValue::Qualify( if (ABSL_PREDICT_FALSE(qualifiers.empty())) { return absl::InvalidArgumentError("invalid select qualifier path."); } - ParsedMessageValueQualifyState qualify_state(value_, descriptor_pool, - message_factory, arena); + ParsedMessageValueQualifyState qualify_state( + value_, descriptor_pool, message_factory, arena, is_unsafe()); for (int i = 0; i < qualifiers.size() - 1; i++) { const auto& qualifier = qualifiers[i]; CEL_RETURN_IF_ERROR(qualify_state.ApplySelectQualifier( diff --git a/common/values/parsed_message_value_test.cc b/common/values/parsed_message_value_test.cc index 1a6c3f628..6a1e59ead 100644 --- a/common/values/parsed_message_value_test.cc +++ b/common/values/parsed_message_value_test.cc @@ -19,6 +19,7 @@ #include "absl/status/status_matchers.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" +#include "base/attribute.h" #include "common/memory.h" #include "common/type.h" #include "common/value.h" @@ -122,5 +123,100 @@ TEST_F(ParsedMessageValueTest, GetFieldByNumber) { IsOkAndHolds(BoolValueIs(false))); } +// A message that is not owned by the evaluation arena, wrapped via the unsafe +// (borrowing) API, must not be deep-copied onto the arena when fields are +// selected through `Qualify()`. +TEST_F(ParsedMessageValueTest, QualifyUnsafeBorrowsMessageField) { + TestAllTypesProto3 message; // Heap-allocated, not on `arena()`. + message.mutable_standalone_message()->set_bb(42); + Value wrapped = Value::WrapMessageUnsafe(&message, descriptor_pool(), + message_factory(), arena()); + ASSERT_TRUE(wrapped.IsParsedMessage()); + + const SelectQualifier qualifiers[] = {FieldSpecifier{ + TestAllTypesProto3::kStandaloneMessageFieldNumber, "standalone_message"}}; + Value result; + int count = 0; + ASSERT_THAT(wrapped.GetParsedMessage().Qualify( + qualifiers, /*presence_test=*/false, descriptor_pool(), + message_factory(), arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_EQ(result.GetParsedMessage().message(), &message.standalone_message()); + + ParsedMessageValue safe_wrapped(&message, arena()); + ASSERT_THAT(safe_wrapped.Qualify(qualifiers, /*presence_test=*/false, + descriptor_pool(), message_factory(), + arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_NE(result.GetParsedMessage().message(), &message.standalone_message()); + EXPECT_EQ(result.GetParsedMessage().message()->GetArena(), arena()); +} + +TEST_F(ParsedMessageValueTest, QualifyUnsafeBorrowsRepeatedMessageField) { + TestAllTypesProto3 message; // Heap-allocated, not on `arena()`. + message.add_repeated_nested_message()->set_bb(42); + Value wrapped = Value::WrapMessageUnsafe(&message, descriptor_pool(), + message_factory(), arena()); + ASSERT_TRUE(wrapped.IsParsedMessage()); + + const SelectQualifier qualifiers[] = { + FieldSpecifier{TestAllTypesProto3::kRepeatedNestedMessageFieldNumber, + "repeated_nested_message"}, + AttributeQualifier::OfInt(0)}; + Value result; + int count = 0; + ASSERT_THAT(wrapped.GetParsedMessage().Qualify( + qualifiers, /*presence_test=*/false, descriptor_pool(), + message_factory(), arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_EQ(result.GetParsedMessage().message(), + &message.repeated_nested_message(0)); + + ParsedMessageValue safe_wrapped(&message, arena()); + ASSERT_THAT(safe_wrapped.Qualify(qualifiers, /*presence_test=*/false, + descriptor_pool(), message_factory(), + arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_NE(result.GetParsedMessage().message(), + &message.repeated_nested_message(0)); + EXPECT_EQ(result.GetParsedMessage().message()->GetArena(), arena()); +} + +TEST_F(ParsedMessageValueTest, QualifyUnsafeBorrowsMapMessageField) { + TestAllTypesProto3 message; // Heap-allocated, not on `arena()`. + (*message.mutable_map_string_message())["key"].set_bb(42); + Value wrapped = Value::WrapMessageUnsafe(&message, descriptor_pool(), + message_factory(), arena()); + ASSERT_TRUE(wrapped.IsParsedMessage()); + + const SelectQualifier qualifiers[] = { + FieldSpecifier{TestAllTypesProto3::kMapStringMessageFieldNumber, + "map_string_message"}, + AttributeQualifier::OfString("key")}; + Value result; + int count = 0; + ASSERT_THAT(wrapped.GetParsedMessage().Qualify( + qualifiers, /*presence_test=*/false, descriptor_pool(), + message_factory(), arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_EQ(result.GetParsedMessage().message(), + &message.map_string_message().at("key")); + + ParsedMessageValue safe_wrapped(&message, arena()); + ASSERT_THAT(safe_wrapped.Qualify(qualifiers, /*presence_test=*/false, + descriptor_pool(), message_factory(), + arena(), &result, &count), + IsOk()); + ASSERT_TRUE(result.IsParsedMessage()); + EXPECT_NE(result.GetParsedMessage().message(), + &message.map_string_message().at("key")); + EXPECT_EQ(result.GetParsedMessage().message()->GetArena(), arena()); +} + } // namespace } // namespace cel From 2b3f8f1f2fba4075a005540697f9f3bdbe801287 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 30 Sep 2026 14:32:16 -0700 Subject: [PATCH 249/350] Add instructions for boolean logic steps (&&/||/!/@not_strictly_false) PiperOrigin-RevId: 991225263 --- eval/compiler/BUILD | 1 - eval/compiler/flat_expr_builder.cc | 10 +- eval/eval/BUILD | 36 +--- eval/eval/evaluator_core.cc | 13 ++ eval/eval/evaluator_core.h | 32 +++ eval/eval/logic_step.cc | 305 ++++++++++++----------------- eval/eval/logic_step.h | 29 +-- eval/eval/logic_step_test.cc | 45 ++++- 8 files changed, 236 insertions(+), 235 deletions(-) diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 76a20f5a8..032a12c74 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -123,7 +123,6 @@ cc_library( "//eval/eval:function_step", "//eval/eval:ident_step", "//eval/eval:jump_step", - "//eval/eval:logic_step", "//eval/eval:optional_or_step", "//eval/eval:select_step", "//eval/eval:shadowable_value_step", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index ef36d6e94..6c82eb912 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -2067,7 +2067,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNot( *depth + 1); return CallHandlerResult::kIntercepted; } - AddStep(CreateNotStep(), expr.id()); + AddStep(ExpressionStep::MakeBooleanNotStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -2092,7 +2092,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNotStrictlyFalse( *depth + 1); return CallHandlerResult::kIntercepted; } - AddStep(CreateNotStrictlyFalseStep(), expr.id()); + AddStep(ExpressionStep::MakeNotStrictlyFalseStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -2257,9 +2257,11 @@ void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { const size_t num_args = expr->call_expr().args().size(); if (arg_num == last_arg_index) { if (is_or_) { - visitor_->AddStep(CreateOrStep(num_args), expr->id()); + visitor_->AddStep( + ExpressionStep::MakeBooleanOrStep(num_args, expr->id())); } else { - visitor_->AddStep(CreateAndStep(num_args), expr->id()); + visitor_->AddStep( + ExpressionStep::MakeBooleanAndStep(num_args, expr->id())); } if (short_circuiting_ && !jump_steps_.empty()) { for (auto& jump : jump_steps_) { diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 41aca62b9..8a1cc795e 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -37,10 +37,12 @@ cc_library( srcs = [ "evaluator_core.cc", "lazy_init_step.cc", + "logic_step.cc", ], hdrs = [ "evaluator_core.h", "lazy_init_step.h", + "logic_step.h", ], deps = [ ":attribute_trail", @@ -49,15 +51,19 @@ cc_library( ":direct_expression_step", ":evaluator_stack", ":iterator_stack", + "//base:builtins", "//base:data", + "//common:casting", "//common:native_type", "//common:value", "//common:value_kind", + "//eval/internal:errors", "//internal:status_macros", "//runtime", "//runtime:activation_interface", "//runtime:runtime_options", "//runtime/internal:activation_attribute_matcher_access", + "//runtime/internal:errors", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", @@ -65,6 +71,7 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:value_cc_proto", "@com_google_protobuf//:protobuf", @@ -430,33 +437,6 @@ cc_library( ], ) -cc_library( - name = "logic_step", - srcs = [ - "logic_step.cc", - ], - hdrs = [ - "logic_step.h", - ], - deps = [ - ":attribute_trail", - ":direct_expression_step", - ":evaluator_core", - ":expression_step_base", - "//base:builtins", - "//common:casting", - "//common:value", - "//common:value_kind", - "//eval/internal:errors", - "//internal:status_macros", - "//runtime/internal:errors", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", - ], -) - cc_library( name = "equality_steps", srcs = [ @@ -760,7 +740,6 @@ cc_test( ":direct_expression_step", ":evaluator_core", ":ident_step", - ":logic_step", "//base:attributes", "//base:data", "//common:casting", @@ -1175,6 +1154,7 @@ cc_test( srcs = ["lazy_init_step_test.cc"], deps = [ ":comprehension_slots", + ":const_value_step", ":evaluator_core", "//base:data", "//common:value", diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index ba275ec80..86ae84d3b 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -31,6 +31,7 @@ #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/lazy_init_step.h" +#include "eval/eval/logic_step.h" #include "internal/status_macros.h" #include "runtime/activation_interface.h" #include "google/protobuf/arena.h" @@ -145,6 +146,18 @@ void ExpressionStep::Evaluate(ExecutionFrame* context) const { case ExpressionStepKind::kClearSlots: EvaluateClearSlotStep(u_.clear_slots, *context); break; + case ExpressionStepKind::kBooleanNot: + EvaluateNotStep(*context); + break; + case ExpressionStepKind::kNotStrictlyFalse: + EvaluateNotStrictlyFalseStep(*context); + break; + case ExpressionStepKind::kBooleanOr: + EvaluateBoolLogicStep(BoolLogicKind::kOr, u_.arg_count, *context); + break; + case ExpressionStepKind::kBooleanAnd: + EvaluateBoolLogicStep(BoolLogicKind::kAnd, u_.arg_count, *context); + break; case ExpressionStepKind::kMovedFrom: default: context->Abort( diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index dea5f527f..73772845c 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -39,6 +39,7 @@ #include "eval/eval/evaluator_stack.h" #include "eval/eval/iterator_stack.h" #include "eval/eval/lazy_init_step.h" +#include "eval/eval/logic_step.h" #include "runtime/activation_interface.h" #include "runtime/internal/activation_attribute_matcher_access.h" #include "runtime/runtime.h" @@ -73,6 +74,10 @@ enum class ExpressionStepKind : uint16_t { kLazyInit = 8, kAssignSlotAndPop = 9, kClearSlots = 10, + kBooleanNot = 11, + kNotStrictlyFalse = 12, + kBooleanOr = 13, + kBooleanAnd = 14, }; class ExpressionStep { @@ -143,6 +148,28 @@ class ExpressionStep { return step; } + static ExpressionStep MakeBooleanNotStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kBooleanNot, id); + } + + static ExpressionStep MakeNotStrictlyFalseStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kNotStrictlyFalse, id); + } + + static ExpressionStep MakeBooleanOrStep(size_t num_args, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBooleanOr, id); + ABSL_DCHECK_LT(num_args, std::numeric_limits::max()); + step.u_.arg_count = num_args; + return step; + } + + static ExpressionStep MakeBooleanAndStep(size_t num_args, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBooleanAnd, id); + ABSL_DCHECK_LT(num_args, std::numeric_limits::max()); + step.u_.arg_count = num_args; + return step; + } + private: struct Header { ExpressionStepKind kind; @@ -189,6 +216,7 @@ class ExpressionStep { LazyInitStepInfo lazy_init; size_t slot_index; ClearSlotStepInfo clear_slots; + size_t arg_count; Data() : empty(nullptr) {} ~Data() {} @@ -731,6 +759,10 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kLazyInit: case ExpressionStepKind::kAssignSlotAndPop: case ExpressionStepKind::kClearSlots: + case ExpressionStepKind::kBooleanNot: + case ExpressionStepKind::kNotStrictlyFalse: + case ExpressionStepKind::kBooleanOr: + case ExpressionStepKind::kBooleanAnd: break; default: ABSL_UNREACHABLE(); diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index 9eaa94f4b..a7528687b 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -7,8 +7,7 @@ #include #include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/types/optional.h" +#include "absl/strings/string_view.h" #include "absl/types/span.h" #include "base/builtins.h" #include "common/casting.h" @@ -17,7 +16,6 @@ #include "eval/eval/attribute_trail.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" #include "eval/internal/errors.h" #include "internal/status_macros.h" #include "runtime/internal/errors.h" @@ -35,11 +33,9 @@ using ::cel::Value; using ::cel::ValueKind; using ::cel::runtime_internal::CreateNoMatchingOverloadError; -enum class OpType { kAnd, kOr }; - // Shared logic for the fall through case (we didn't see the shortcircuit // value). -absl::Status ReturnLogicResult(ExecutionFrameBase& frame, OpType op_type, +absl::Status ReturnLogicResult(ExecutionFrameBase& frame, BoolLogicKind op_type, Value& lhs_result, Value& rhs_result, AttributeTrail& attribute_trail, AttributeTrail& rhs_attr) { @@ -78,8 +74,9 @@ absl::Status ReturnLogicResult(ExecutionFrameBase& frame, OpType op_type, // Otherwise, add a no overload error. attribute_trail = AttributeTrail(); lhs_result = cel::ErrorValue::From( - CreateNoMatchingOverloadError( - op_type == OpType::kOr ? cel::builtin::kOr : cel::builtin::kAnd), + CreateNoMatchingOverloadError(op_type == BoolLogicKind::kOr + ? cel::builtin::kOr + : cel::builtin::kAnd), frame.arena()); return absl::OkStatus(); } @@ -88,7 +85,7 @@ class ExhaustiveDirectLogicStep : public DirectExpressionStep { public: explicit ExhaustiveDirectLogicStep(std::unique_ptr lhs, std::unique_ptr rhs, - OpType op_type, int64_t expr_id) + BoolLogicKind op_type, int64_t expr_id) : DirectExpressionStep(expr_id), lhs_(std::move(lhs)), rhs_(std::move(rhs)), @@ -100,7 +97,7 @@ class ExhaustiveDirectLogicStep : public DirectExpressionStep { private: std::unique_ptr lhs_; std::unique_ptr rhs_; - OpType op_type_; + BoolLogicKind op_type_; }; absl::Status ExhaustiveDirectLogicStep::Evaluate( @@ -116,16 +113,16 @@ absl::Status ExhaustiveDirectLogicStep::Evaluate( ValueKind rhs_kind = rhs_result.kind(); if (lhs_kind == ValueKind::kBool) { bool lhs_bool = Cast(result).NativeValue(); - if ((op_type_ == OpType::kOr && lhs_bool) || - (op_type_ == OpType::kAnd && !lhs_bool)) { + if ((op_type_ == BoolLogicKind::kOr && lhs_bool) || + (op_type_ == BoolLogicKind::kAnd && !lhs_bool)) { return absl::OkStatus(); } } if (rhs_kind == ValueKind::kBool) { bool rhs_bool = Cast(rhs_result).NativeValue(); - if ((op_type_ == OpType::kOr && rhs_bool) || - (op_type_ == OpType::kAnd && !rhs_bool)) { + if ((op_type_ == BoolLogicKind::kOr && rhs_bool) || + (op_type_ == BoolLogicKind::kAnd && !rhs_bool)) { result = std::move(rhs_result); attribute_trail = std::move(rhs_attr); return absl::OkStatus(); @@ -140,7 +137,7 @@ class DirectLogicStep : public DirectExpressionStep { public: explicit DirectLogicStep(std::unique_ptr lhs, std::unique_ptr rhs, - OpType op_type, int64_t expr_id) + BoolLogicKind op_type, int64_t expr_id) : DirectExpressionStep(expr_id), lhs_(std::move(lhs)), rhs_(std::move(rhs)), @@ -152,7 +149,7 @@ class DirectLogicStep : public DirectExpressionStep { private: std::unique_ptr lhs_; std::unique_ptr rhs_; - OpType op_type_; + BoolLogicKind op_type_; }; absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, @@ -161,8 +158,8 @@ absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, ValueKind lhs_kind = result.kind(); if (lhs_kind == ValueKind::kBool) { bool lhs_bool = Cast(result).NativeValue(); - if ((op_type_ == OpType::kOr && lhs_bool) || - (op_type_ == OpType::kAnd && !lhs_bool)) { + if ((op_type_ == BoolLogicKind::kOr && lhs_bool) || + (op_type_ == BoolLogicKind::kAnd && !lhs_bool)) { return absl::OkStatus(); } } @@ -176,8 +173,8 @@ absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, if (rhs_kind == ValueKind::kBool) { bool rhs_bool = Cast(rhs_result).NativeValue(); - if ((op_type_ == OpType::kOr && rhs_bool) || - (op_type_ == OpType::kAnd && !rhs_bool)) { + if ((op_type_ == BoolLogicKind::kOr && rhs_bool) || + (op_type_ == BoolLogicKind::kAnd && !rhs_bool)) { result = std::move(rhs_result); attribute_trail = std::move(rhs_attr); return absl::OkStatus(); @@ -188,94 +185,10 @@ absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, rhs_attr); } -class LogicalOpStep : public ExpressionStepBase { - public: - // Constructs FunctionStep that uses overloads specified. - LogicalOpStep(OpType op_type, size_t count) - : shortcircuit_(op_type == OpType::kOr), - op_type_(op_type), - count_(count) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - private: - void Calculate(ExecutionFrame* frame, absl::Span args, - Value& result) const { - std::optional error_pos; - - for (size_t i = 0; i < args.size(); i++) { - const Value& arg = args[i]; - switch (arg.kind()) { - case ValueKind::kBool: - if (arg.GetBool() == shortcircuit_) { - result = arg; - return; - } - break; - case ValueKind::kUnknown: - break; - case ValueKind::kError: - default: - if (!error_pos.has_value()) { - error_pos = i; - } - break; - } - } - - // As opposed to regular function, logical operation treat Unknowns with - // higher precedence than error. This is due to the fact that after Unknown - // is resolved to actual value, it may short-circuit and thus hide the - // error. - if (frame->enable_unknowns()) { - // Check if unknown? - absl::optional unknown_set = - frame->attribute_utility().MergeUnknowns(args); - if (unknown_set.has_value()) { - result = std::move(*unknown_set); - return; - } - } - - if (!error_pos.has_value()) { - result = cel::BoolValue(!shortcircuit_); - return; - } - - result = args[error_pos.value()]; - if (!result.IsError()) { - result = cel::ErrorValue::From( - CreateNoMatchingOverloadError((op_type_ == OpType::kOr) - ? cel::builtin::kOr - : cel::builtin::kAnd), - frame->arena()); - } - } - - bool shortcircuit_; - const OpType op_type_; - size_t count_; -}; - -absl::Status LogicalOpStep::Evaluate(ExecutionFrame* frame) const { - // Must have 2 or more values on the stack. - if (!frame->value_stack().HasEnough(count_)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - // Create Span object that contains input arguments to the function. - auto args = frame->value_stack().GetSpan(count_); - Value result; - Calculate(frame, args, result); - frame->value_stack().PopAndPush(args.size(), std::move(result)); - - return absl::OkStatus(); -} - std::unique_ptr CreateDirectLogicStep( std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, OpType op_type, - bool shortcircuiting) { + std::unique_ptr rhs, int64_t expr_id, + BoolLogicKind op_type, bool shortcircuiting) { if (shortcircuiting) { return std::make_unique(std::move(lhs), std::move(rhs), op_type, expr_id); @@ -326,47 +239,6 @@ absl::Status DirectNotStep::Evaluate(ExecutionFrameBase& frame, Value& result, return absl::OkStatus(); } -class IterativeNotStep : public ExpressionStepBase { - public: - IterativeNotStep() = default; - - absl::Status Evaluate(ExecutionFrame* frame) const override; -}; - -absl::Status IterativeNotStep::Evaluate(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError("Value stack underflow"); - } - const Value& operand = frame->value_stack().Peek(); - - if (frame->unknown_processing_enabled()) { - const AttributeTrail& attribute_trail = - frame->value_stack().PeekAttribute(); - if (frame->attribute_utility().CheckForUnknownPartial(attribute_trail)) { - frame->value_stack().PopAndPush( - frame->attribute_utility().CreateUnknownSet( - attribute_trail.attribute())); - return absl::OkStatus(); - } - } - - switch (operand.kind()) { - case ValueKind::kBool: - frame->value_stack().PopAndPush( - BoolValue{!operand.GetBool().NativeValue()}); - break; - case ValueKind::kUnknown: - case ValueKind::kError: - // just forward. - break; - default: - frame->value_stack().PopAndPush(cel::ErrorValue::From( - CreateNoMatchingOverloadError(cel::builtin::kNot), frame->arena())); - break; - } - - return absl::OkStatus(); -} class DirectNotStrictlyFalseStep : public DirectExpressionStep { public: @@ -402,45 +274,128 @@ absl::Status DirectNotStrictlyFalseStep::Evaluate( return absl::OkStatus(); } -class IterativeNotStrictlyFalseStep : public ExpressionStepBase { - public: - IterativeNotStrictlyFalseStep() = default; +} // namespace - absl::Status Evaluate(ExecutionFrame* frame) const override; -}; +void EvaluateNotStep(ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(1)) { + frame.Abort(absl::InternalError("Value stack underflow")); + return; + } + const Value& operand = frame.value_stack().Peek(); -absl::Status IterativeNotStrictlyFalseStep::Evaluate( - ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError("Value stack underflow"); + if (frame.unknown_processing_enabled()) { + const AttributeTrail& attribute_trail = frame.value_stack().PeekAttribute(); + if (frame.attribute_utility().CheckForUnknownPartial(attribute_trail)) { + frame.value_stack().PopAndPush(frame.attribute_utility().CreateUnknownSet( + attribute_trail.attribute())); + return; + } } - const Value& operand = frame->value_stack().Peek(); switch (operand.kind()) { case ValueKind::kBool: - // just forward. + frame.value_stack().PopAndPush( + BoolValue{!operand.GetBool().NativeValue()}); break; case ValueKind::kUnknown: case ValueKind::kError: - frame->value_stack().PopAndPush(BoolValue(true)); + // just forward. break; default: - frame->value_stack().PopAndPush(cel::ErrorValue::From( - CreateNoMatchingOverloadError(cel::builtin::kNot), frame->arena())); + frame.value_stack().PopAndPush(cel::ErrorValue::From( + CreateNoMatchingOverloadError(cel::builtin::kNot), frame.arena())); break; } +} - return absl::OkStatus(); +void EvaluateNotStrictlyFalseStep(ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(1)) { + frame.Abort(absl::InternalError("Value stack underflow")); + return; + } + const Value& operand = frame.value_stack().Peek(); + + switch (operand.kind()) { + case ValueKind::kBool: + // just forward. + break; + case ValueKind::kUnknown: + case ValueKind::kError: + frame.value_stack().PopAndPush(BoolValue(true)); + break; + default: + frame.value_stack().PopAndPush(cel::ErrorValue::From( + CreateNoMatchingOverloadError(cel::builtin::kNot), frame.arena())); + break; + } } -} // namespace +void EvaluateBoolLogicStep(BoolLogicKind kind, size_t num_args, + ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(num_args)) { + frame.Abort(absl::InternalError("Value stack underflow")); + return; + } + + const bool shortcircuit = kind == BoolLogicKind::kOr; + const absl::string_view op_name = + kind == BoolLogicKind::kOr ? cel::builtin::kOr : cel::builtin::kAnd; + absl::Span args = frame.value_stack().GetSpan(num_args); + std::optional error_pos; + + for (size_t i = 0; i < args.size(); i++) { + const Value& arg = args[i]; + switch (arg.kind()) { + case ValueKind::kBool: + if (arg.GetBool() == shortcircuit) { + frame.value_stack().PopAndPush(num_args, + cel::BoolValue(shortcircuit)); + return; + } + break; + case ValueKind::kUnknown: + break; + case ValueKind::kError: + default: + if (!error_pos.has_value()) { + error_pos = i; + } + break; + } + } + + // As opposed to regular function, logical operation treat Unknowns with + // higher precedence than error. This is due to the fact that after Unknown + // is resolved to actual value, it may short-circuit and thus hide the + // error. + if (frame.enable_unknowns()) { + // Check if unknown? + std::optional unknown_set = + frame.attribute_utility().MergeUnknowns(args); + if (unknown_set.has_value()) { + frame.value_stack().PopAndPush(num_args, *std::move(unknown_set)); + return; + } + } + + if (!error_pos.has_value()) { + frame.value_stack().PopAndPush(num_args, cel::BoolValue(!shortcircuit)); + return; + } + + cel::Value result = args[error_pos.value()]; + if (!result.IsError()) { + result = cel::ErrorValue(CreateNoMatchingOverloadError(op_name)); + } + frame.value_stack().PopAndPush(num_args, std::move(result)); +} std::unique_ptr CreateDirectAndStep( std::unique_ptr lhs, std::unique_ptr rhs, int64_t expr_id, bool shortcircuiting) { return CreateDirectLogicStep(std::move(lhs), std::move(rhs), expr_id, - OpType::kAnd, shortcircuiting); + BoolLogicKind::kAnd, shortcircuiting); } std::unique_ptr CreateDirectOrStep( @@ -448,17 +403,7 @@ std::unique_ptr CreateDirectOrStep( std::unique_ptr rhs, int64_t expr_id, bool shortcircuiting) { return CreateDirectLogicStep(std::move(lhs), std::move(rhs), expr_id, - OpType::kOr, shortcircuiting); -} - -// Factory method for "And" Execution step -std::unique_ptr CreateAndStep(size_t num_args) { - return std::make_unique(OpType::kAnd, num_args); -} - -// Factory method for "Or" Execution step -std::unique_ptr CreateOrStep(size_t num_args) { - return std::make_unique(OpType::kOr, num_args); + BoolLogicKind::kOr, shortcircuiting); } // Factory method for recursive logical not "!" Execution step @@ -467,11 +412,6 @@ std::unique_ptr CreateDirectNotStep( return std::make_unique(std::move(operand), expr_id); } -// Factory method for iterative logical not "!" Execution step -std::unique_ptr CreateNotStep() { - return std::make_unique(); -} - // Factory method for recursive logical "@not_strictly_false" Execution step. std::unique_ptr CreateDirectNotStrictlyFalseStep( std::unique_ptr operand, int64_t expr_id) { @@ -479,9 +419,4 @@ std::unique_ptr CreateDirectNotStrictlyFalseStep( expr_id); } -// Factory method for iterative logical "@not_strictly_false" Execution step. -std::unique_ptr CreateNotStrictlyFalseStep() { - return std::make_unique(); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/logic_step.h b/eval/eval/logic_step.h index 95f58b9db..401eaf2ca 100644 --- a/eval/eval/logic_step.h +++ b/eval/eval/logic_step.h @@ -1,15 +1,28 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_LOGIC_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_LOGIC_STEP_H_ +#include #include #include -#include "absl/status/statusor.h" #include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { +class ExecutionFrame; + +void EvaluateNotStep(ExecutionFrame& frame); + +void EvaluateNotStrictlyFalseStep(ExecutionFrame& frame); + +enum class BoolLogicKind { + kAnd, + kOr, +}; + +void EvaluateBoolLogicStep(BoolLogicKind kind, size_t num_args, + ExecutionFrame& frame); + // Factory method for "And" Execution step std::unique_ptr CreateDirectAndStep( std::unique_ptr lhs, @@ -22,26 +35,14 @@ std::unique_ptr CreateDirectOrStep( std::unique_ptr rhs, int64_t expr_id, bool shortcircuiting); -// Factory method for "And" Execution step -std::unique_ptr CreateAndStep(size_t num_args); - -// Factory method for "Or" Execution step -std::unique_ptr CreateOrStep(size_t num_args); - // Factory method for recursive logical not "!" Execution step std::unique_ptr CreateDirectNotStep( std::unique_ptr operand, int64_t expr_id); -// Factory method for iterative logical not "!" Execution step -std::unique_ptr CreateNotStep(); - // Factory method for recursive logical "@not_strictly_false" Execution step. std::unique_ptr CreateDirectNotStrictlyFalseStep( std::unique_ptr operand, int64_t expr_id); -// Factory method for iterative logical "@not_strictly_false" Execution step. -std::unique_ptr CreateNotStrictlyFalseStep(); - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_LOGIC_STEP_H_ diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index bdb6ceed9..93f2fb888 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -69,9 +69,9 @@ class LogicStepTest : public testing::TestWithParam { ExecutionPath path; path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name0"))); path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name1"))); - path.push_back(ExpressionStep::MakeGenericStep( - (is_or) ? CreateOrStep(/*num_args=*/2) : CreateAndStep(/*num_args=*/2), - /*expr_id=*/2)); + path.push_back( + (is_or) ? ExpressionStep::MakeBooleanOrStep(/*num_args=*/2, /*id=*/2) + : ExpressionStep::MakeBooleanAndStep(/*num_args=*/2, /*id=*/2)); auto dummy_expr = std::make_unique(); cel::RuntimeOptions options; @@ -642,6 +642,45 @@ INSTANTIATE_TEST_SUITE_P( [](const testing::TestParamInfo& info) -> std::string { return info.param.name; }); +TEST(UnaryLogicStepTest, BooleanNot) { + ExecutionPath path; + path.push_back(ExpressionStep::MakeConstant(cel::BoolValue(true))); + path.push_back(ExpressionStep::MakeBooleanNotStep()); + + google::protobuf::Arena arena; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + FlatExpressionEvaluatorState state( + 2, 0, type_provider, cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + cel::Activation activation; + cel::RuntimeOptions options; + ExecutionFrame frame(path, activation, options, state); + ASSERT_OK_AND_ASSIGN(cel::Value value, frame.Evaluate()); + ASSERT_TRUE(value.IsBool()); + EXPECT_FALSE(value.GetBool().NativeValue()); +} + +TEST(UnaryLogicStepTest, NotStrictlyFalse) { + ExecutionPath path; + path.push_back(ExpressionStep::MakeConstant( + cel::ErrorValue(absl::InternalError("error")))); + path.push_back(ExpressionStep::MakeNotStrictlyFalseStep()); + + google::protobuf::Arena arena; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + FlatExpressionEvaluatorState state( + 2, 0, type_provider, cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + cel::Activation activation; + cel::RuntimeOptions options; + ExecutionFrame frame(path, activation, options, state); + ASSERT_OK_AND_ASSIGN(cel::Value value, frame.Evaluate()); + ASSERT_TRUE(value.IsBool()); + EXPECT_TRUE(value.GetBool().NativeValue()); +} + } // namespace } // namespace google::api::expr::runtime From c6f0940f386baf55b15aefd4921a7cdfc7ce1437 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 30 Sep 2026 16:35:28 -0700 Subject: [PATCH 250/350] Add instructions for comprehension steps. PiperOrigin-RevId: 991291484 --- eval/compiler/BUILD | 3 +- eval/compiler/flat_expr_builder.cc | 123 ++++++---- eval/compiler/flat_expr_builder_extensions.h | 14 +- eval/eval/BUILD | 52 ++--- eval/eval/comprehension_step.cc | 231 ++++++++++++------- eval/eval/comprehension_step.h | 79 +++---- eval/eval/comprehension_step_test.cc | 17 +- eval/eval/evaluator_core.cc | 51 +++- eval/eval/evaluator_core.h | 103 +++++++-- eval/eval/expression_step_base.h | 2 +- eval/eval/expression_step_logic.h | 50 ++++ eval/eval/ident_step.cc | 24 +- eval/eval/ident_step.h | 7 +- eval/eval/iterator_stack.h | 48 +++- 14 files changed, 517 insertions(+), 287 deletions(-) create mode 100644 eval/eval/expression_step_logic.h diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 032a12c74..541eb4d79 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -36,6 +36,7 @@ cc_library( "//common:value", "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", + "//eval/eval:expression_step_logic", "//eval/eval:trace_step", "//internal:casts", "//runtime:runtime_options", @@ -111,7 +112,6 @@ cc_library( "//common:type", "//common:type_spec_resolver", "//common:value", - "//eval/eval:comprehension_step", "//eval/eval:const_value_step", "//eval/eval:container_access_step", "//eval/eval:create_list_step", @@ -120,6 +120,7 @@ cc_library( "//eval/eval:direct_expression_step", "//eval/eval:equality_steps", "//eval/eval:evaluator_core", + "//eval/eval:expression_step_logic", "//eval/eval:function_step", "//eval/eval:ident_step", "//eval/eval:jump_step", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 6c82eb912..514e28df4 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -31,6 +31,7 @@ #include #include "absl/algorithm/container.h" +#include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/container/node_hash_map.h" @@ -71,6 +72,7 @@ #include "eval/eval/direct_expression_step.h" #include "eval/eval/equality_steps.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "eval/eval/function_step.h" #include "eval/eval/ident_step.h" #include "eval/eval/jump_step.h" @@ -427,13 +429,11 @@ bool IsBlock(const cel::CallExpr* call) { return call->function() == kBlock; } // Visitor for Comprehension expressions. class ComprehensionVisitor { public: - explicit ComprehensionVisitor(FlatExprVisitor* visitor, bool short_circuiting, - bool is_trivial, size_t iter_slot, - size_t iter2_slot, size_t accu_slot) + explicit ComprehensionVisitor(FlatExprVisitor* visitor, bool is_trivial, + size_t iter_slot, size_t iter2_slot, + size_t accu_slot) : visitor_(visitor), - next_step_(nullptr), - cond_step_(nullptr), - short_circuiting_(short_circuiting), + init_step_(nullptr), is_trivial_(is_trivial), accu_init_extracted_(false), iter_slot_(iter_slot), @@ -461,14 +461,14 @@ class ComprehensionVisitor { absl::Status PostVisitArgDefault(cel::ComprehensionArg arg_num, const cel::Expr* comprehension_expr); + ComprehensionCondStep* absl_nullable GetCondStep(); + ComprehensionNextStep* absl_nullable GetNextStep(); + FlatExprVisitor* visitor_; ComprehensionInitStep* init_step_; - ComprehensionNextStep* next_step_; - ComprehensionCondStep* cond_step_; ProgramStepIndex init_step_pos_; - ProgramStepIndex next_step_pos_; - ProgramStepIndex cond_step_pos_; - bool short_circuiting_; + std::optional next_step_pos_; + std::optional cond_step_pos_; bool is_trivial_; bool accu_init_extracted_; size_t iter_slot_; @@ -865,8 +865,7 @@ class FlatExprVisitor : public cel::AstVisitor { CreateDirectSlotIdentStep(ident_expr.name(), slot.slot, expr.id()), 1); } else { - AddStep(CreateIdentStepForSlot(ident_expr.name(), slot.slot), - expr.id()); + AddStep(ExpressionStep::MakeReadSlotStep(slot.slot, expr.id())); } return; } @@ -1439,9 +1438,8 @@ class FlatExprVisitor : public cel::AstVisitor { /*.iter_var2_in_scope=*/false, /*.accu_var_in_scope=*/false, /*.in_accu_init=*/false, - std::make_unique(this, options_.short_circuiting, - is_bind, iter_slot, iter2_slot, - accu_slot)}); + std::make_unique(this, is_bind, iter_slot, + iter2_slot, accu_slot)}); comprehension_stack_.back().visitor->PreVisit(&expr); } @@ -2433,6 +2431,30 @@ void ComprehensionVisitor::PreVisit(const cel::Expr* expr) { } } +ComprehensionCondStep* absl_nullable ComprehensionVisitor::GetCondStep() { + if (!cond_step_pos_) { + return nullptr; + } + ExpressionStep* step = + cond_step_pos_->subexpression->GetIfExpressionStep(cond_step_pos_->index); + if (!step) { + return nullptr; + } + return GetIfComprehensionCondStep(*step); +} + +ComprehensionNextStep* absl_nullable ComprehensionVisitor::GetNextStep() { + if (!next_step_pos_) { + return nullptr; + } + ExpressionStep* step = + next_step_pos_->subexpression->GetIfExpressionStep(next_step_pos_->index); + if (!step) { + return nullptr; + } + return GetIfComprehensionNextStep(*step); +} + absl::Status ComprehensionVisitor::PostVisitArgDefault( cel::ComprehensionArg arg_num, const cel::Expr* expr) { if (visitor_->PlanRecursiveProgram()) { @@ -2441,19 +2463,31 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( switch (arg_num) { case cel::ITER_RANGE: { init_step_pos_ = visitor_->GetCurrentIndex(); - init_step_ = visitor_->AddStep(std::make_unique()); + if (iter_slot_ != iter2_slot_) { + init_step_ = visitor_->AddStep(std::make_unique( + iter_slot_, iter2_slot_, accu_slot_)); + } else { + init_step_ = visitor_->AddStep( + std::make_unique(iter_slot_, accu_slot_)); + } break; } case cel::ACCU_INIT: { next_step_pos_ = visitor_->GetCurrentIndex(); - next_step_ = visitor_->AddStep(std::make_unique( - iter_slot_, iter2_slot_, accu_slot_)); + if (iter_slot_ != iter2_slot_) { + visitor_->AddStep(ExpressionStep::MakeComprehensionNext2Step()); + } else { + visitor_->AddStep(ExpressionStep::MakeComprehensionNextStep()); + } break; } case cel::LOOP_CONDITION: { cond_step_pos_ = visitor_->GetCurrentIndex(); - cond_step_ = visitor_->AddStep(std::make_unique( - iter_slot_, iter2_slot_, accu_slot_, short_circuiting_)); + if (iter_slot_ != iter2_slot_) { + visitor_->AddStep(ExpressionStep::MakeComprehensionCond2Step()); + } else { + visitor_->AddStep(ExpressionStep::MakeComprehensionCondStep()); + } break; } case cel::LOOP_STEP: { @@ -2464,46 +2498,51 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( } Jump jump_helper(index, jump_to_next); visitor_->SetProgressStatusIfError( - jump_helper.set_target(next_step_pos_)); + jump_helper.set_target(*next_step_pos_)); // Set offsets jumping to the result step. - if (cond_step_) { - CEL_ASSIGN_OR_RETURN( - int jump_from_cond, - Jump::CalculateOffset(cond_step_pos_, visitor_->GetCurrentIndex())); - cond_step_->set_jump_offset(jump_from_cond); + if (auto* cond_step = GetCondStep(); cond_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_cond, + Jump::CalculateOffset( + *cond_step_pos_, visitor_->GetCurrentIndex())); + cond_step->set_jump_offset(jump_from_cond); } - if (next_step_) { - CEL_ASSIGN_OR_RETURN( - int jump_from_next, - Jump::CalculateOffset(next_step_pos_, visitor_->GetCurrentIndex())); + if (auto* next_step = GetNextStep(); next_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_next, + Jump::CalculateOffset( + *next_step_pos_, visitor_->GetCurrentIndex())); - next_step_->set_jump_offset(jump_from_next); + next_step->set_jump_offset(jump_from_next); } break; } case cel::RESULT: { - if (!init_step_ || !next_step_ || !cond_step_) { + if (!init_step_ || !next_step_pos_ || !cond_step_pos_) { // Encountered an error earlier. Can't determine where to jump. break; } - visitor_->AddStep(CreateComprehensionFinishStep(accu_slot_), expr->id()); + visitor_->AddStep( + ExpressionStep::MakeComprehensionFinishStep(accu_slot_, expr->id())); // Set offsets jumping past the result step in case of errors. CEL_ASSIGN_OR_RETURN( int jump_from_init, Jump::CalculateOffset(init_step_pos_, visitor_->GetCurrentIndex())); init_step_->set_error_jump_offset(jump_from_init); - CEL_ASSIGN_OR_RETURN( - int jump_from_next, - Jump::CalculateOffset(next_step_pos_, visitor_->GetCurrentIndex())); - next_step_->set_error_jump_offset(jump_from_next); + if (auto* next_step = GetNextStep(); next_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_next, + Jump::CalculateOffset( + *next_step_pos_, visitor_->GetCurrentIndex())); + next_step->set_error_jump_offset(jump_from_next); + } - CEL_ASSIGN_OR_RETURN( - int jump_from_cond, - Jump::CalculateOffset(cond_step_pos_, visitor_->GetCurrentIndex())); - cond_step_->set_error_jump_offset(jump_from_cond); + if (auto* cond_step = GetCondStep(); cond_step != nullptr) { + CEL_ASSIGN_OR_RETURN(int jump_from_cond, + Jump::CalculateOffset( + *cond_step_pos_, visitor_->GetCurrentIndex())); + cond_step->set_error_jump_offset(jump_from_cond); + } break; } } diff --git a/eval/compiler/flat_expr_builder_extensions.h b/eval/compiler/flat_expr_builder_extensions.h index 2ce70a166..e85f337f9 100644 --- a/eval/compiler/flat_expr_builder_extensions.h +++ b/eval/compiler/flat_expr_builder_extensions.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "absl/base/attributes.h" @@ -45,6 +46,7 @@ #include "eval/compiler/resolver.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "eval/eval/trace_step.h" #include "internal/casts.h" #include "runtime/internal/issue_collector.h" @@ -125,6 +127,13 @@ class ProgramBuilder { elements().push_back(expr); } + ExpressionStep* absl_nullable GetIfExpressionStep(size_t index) { + if (index >= elements().size()) { + return nullptr; + } + return std::get_if(&elements()[index]); + } + // Accessor for elements (either simple steps or subexpressions). // // Value is undefined if in the expression has already been flattened. @@ -288,9 +297,8 @@ class ProgramBuilder { // Add a program step to the current subexpression. // If successful, returns the step pointer. // - // Note: If successful, the pointer should remain valid until the parent - // expression is finalized. Optimizers may modify the program plan which may - // free the step at that point. + // Note: If successful, the pointer should remain valid until a further call + // to AddStep, AddSubexpression, or Flatten. ExpressionStep* absl_nullable AddStep(ExpressionStep step); void Reset(); diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 8a1cc795e..5dfaf69a1 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -35,11 +35,13 @@ package_group( cc_library( name = "evaluator_core", srcs = [ + "comprehension_step.cc", "evaluator_core.cc", "lazy_init_step.cc", "logic_step.cc", ], hdrs = [ + "comprehension_step.h", "evaluator_core.h", "lazy_init_step.h", "logic_step.h", @@ -50,7 +52,9 @@ cc_library( ":comprehension_slots", ":direct_expression_step", ":evaluator_stack", + ":expression_step_logic", ":iterator_stack", + "//base:attributes", "//base:builtins", "//base:data", "//common:casting", @@ -64,6 +68,7 @@ cc_library( "//runtime:runtime_options", "//runtime/internal:activation_attribute_matcher_access", "//runtime/internal:errors", + "@com_google_absl//absl/base", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", @@ -180,12 +185,26 @@ cc_test( ], ) +cc_library( + name = "expression_step_logic", + hdrs = [ + "expression_step_logic.h", + ], + deps = [ + "//common:native_type", + "@com_google_absl//absl/status", + ], +) + cc_library( name = "expression_step_base", hdrs = [ "expression_step_base.h", ], - deps = [":evaluator_core"], + deps = [ + ":evaluator_core", + ":expression_step_logic", + ], ) cc_library( @@ -267,6 +286,7 @@ cc_library( ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:value", "//eval/internal:errors", "//internal:status_macros", @@ -490,35 +510,6 @@ cc_test( ], ) -cc_library( - name = "comprehension_step", - srcs = [ - "comprehension_step.cc", - ], - hdrs = [ - "comprehension_step.h", - ], - deps = [ - ":attribute_trail", - ":comprehension_slots", - ":direct_expression_step", - ":evaluator_core", - ":expression_step_base", - "//base:attributes", - "//common:casting", - "//common:value", - "//common:value_kind", - "//eval/internal:errors", - "//internal:status_macros", - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - ], -) - cc_test( name = "comprehension_step_test", size = "small", @@ -529,7 +520,6 @@ cc_test( ":attribute_trail", ":cel_expression_flat_impl", ":comprehension_slots", - ":comprehension_step", ":const_value_step", ":direct_expression_step", ":evaluator_core", diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc index 7f6d5b4e2..d67a7e8a6 100644 --- a/eval/eval/comprehension_step.cc +++ b/eval/eval/comprehension_step.cc @@ -20,7 +20,7 @@ #include "eval/eval/comprehension_slots.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" +#include "eval/eval/iterator_stack.h" #include "eval/internal/errors.h" #include "internal/status_macros.h" @@ -41,6 +41,7 @@ using ::cel::ValueIterator; using ::cel::ValueIteratorPtr; using ::cel::ValueKind; using ::cel::runtime_internal::CreateNoMatchingOverloadError; +using ::cel::runtime_internal::IteratorStack; AttributeQualifier AttributeQualifierFromValue(const Value& v) { switch (v.kind()) { @@ -58,25 +59,6 @@ AttributeQualifier AttributeQualifierFromValue(const Value& v) { } } -class ComprehensionFinishStep final : public ExpressionStepBase { - public: - explicit ComprehensionFinishStep(size_t accu_slot) - : ExpressionStepBase(), accu_slot_(accu_slot) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - frame->value_stack().SwapAndPop(2, 1); - frame->comprehension_slots().ClearSlot(accu_slot_); - frame->iterator_stack().Pop(); - return absl::OkStatus(); - } - - private: - const size_t accu_slot_; -}; - class ComprehensionDirectStep final : public DirectExpressionStep { public: explicit ComprehensionDirectStep( @@ -468,11 +450,23 @@ absl::Status ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { switch (top.kind()) { case ValueKind::kList: { CEL_ASSIGN_OR_RETURN(auto iterator, top.GetList().NewIterator()); - frame->iterator_stack().Push(std::move(iterator)); + if (has_iter2_) { + frame->iterator_stack().Push(std::move(iterator), iter_slot_, + iter2_slot_, accu_slot_); + } else { + frame->iterator_stack().Push(std::move(iterator), iter_slot_, + accu_slot_); + } } break; case ValueKind::kMap: { CEL_ASSIGN_OR_RETURN(auto iterator, top.GetMap().NewIterator()); - frame->iterator_stack().Push(std::move(iterator)); + if (has_iter2_) { + frame->iterator_stack().Push(std::move(iterator), iter_slot_, + iter2_slot_, accu_slot_); + } else { + frame->iterator_stack().Push(std::move(iterator), iter_slot_, + accu_slot_); + } } break; default: // Replace with an error and jump past @@ -485,21 +479,28 @@ absl::Status ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { return absl::OkStatus(); } -absl::Status ComprehensionNextStep::Evaluate1(ExecutionFrame* frame) const { +void ComprehensionNextStep::Evaluate1(ExecutionFrame* frame) const { + if (frame->iterator_stack().empty()) { + frame->Abort(absl::InternalError("Iterator stack underflow")); + return; + } + const IteratorStack::Entry& entry = *frame->iterator_stack().Peek(); if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + frame->Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; } { Value& accu_var = frame->value_stack().Peek(); AttributeTrail& accu_var_attr = frame->value_stack().PeekAttribute(); - frame->comprehension_slots().Set(accu_slot_, std::move(accu_var), + frame->comprehension_slots().Set(entry.accu_slot, std::move(accu_var), std::move(accu_var_attr)); frame->value_stack().Pop(1); } ComprehensionSlots::Slot* iter_slot = - frame->comprehension_slots().Get(iter_slot_); + frame->comprehension_slots().Get(entry.iter_slot); ABSL_DCHECK(iter_slot != nullptr); iter_slot->Set(); @@ -519,15 +520,23 @@ absl::Status ComprehensionNextStep::Evaluate1(ExecutionFrame* frame) const { default: ABSL_UNREACHABLE(); } - CEL_ASSIGN_OR_RETURN(bool ok, - frame->iterator_stack().Peek()->Next2( - frame->descriptor_pool(), frame->message_factory(), - frame->arena(), key, value)); - if (!ok) { + absl::StatusOr ok = entry.iterator->Next2(frame->descriptor_pool(), + frame->message_factory(), + frame->arena(), key, value); + if (!ok.ok()) { + frame->Abort(std::move(ok).status()); + return; + } + if (!*ok) { iter_slot->Clear(); - return frame->JumpTo(jump_offset_); + frame->JumpToOrAbort(jump_offset_); + return; + } + absl::Status inc_status = frame->IncrementIterations(); + if (!inc_status.ok()) { + frame->Abort(std::move(inc_status)); + return; } - CEL_RETURN_IF_ERROR(frame->IncrementIterations()); *iter_slot->mutable_attribute() = frame->value_stack().PeekAttribute().Step( AttributeQualifierFromValue(*key)); if (frame->attribute_utility().CheckForUnknownExact( @@ -536,53 +545,74 @@ absl::Status ComprehensionNextStep::Evaluate1(ExecutionFrame* frame) const { iter_slot->attribute().attribute()); } } else { - CEL_ASSIGN_OR_RETURN(bool ok, - frame->iterator_stack().Peek()->Next1( - frame->descriptor_pool(), frame->message_factory(), - frame->arena(), iter_slot->mutable_value())); - if (!ok) { + absl::StatusOr ok = entry.iterator->Next1( + frame->descriptor_pool(), frame->message_factory(), frame->arena(), + iter_slot->mutable_value()); + if (!ok.ok()) { + frame->Abort(std::move(ok).status()); + return; + } + if (!*ok) { iter_slot->Clear(); - return frame->JumpTo(jump_offset_); + frame->JumpToOrAbort(jump_offset_); + return; + } + absl::Status inc_status = frame->IncrementIterations(); + if (!inc_status.ok()) { + frame->Abort(std::move(inc_status)); + return; } - CEL_RETURN_IF_ERROR(frame->IncrementIterations()); } - return absl::OkStatus(); } -absl::Status ComprehensionNextStep::Evaluate2(ExecutionFrame* frame) const { +void ComprehensionNextStep::Evaluate2(ExecutionFrame* frame) const { + if (frame->iterator_stack().empty()) { + frame->Abort(absl::InternalError("Iterator stack underflow")); + return; + } + const IteratorStack::Entry& entry = *frame->iterator_stack().Peek(); if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + frame->Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; } { Value& accu_var = frame->value_stack().Peek(); AttributeTrail& accu_var_attr = frame->value_stack().PeekAttribute(); - frame->comprehension_slots().Set(accu_slot_, std::move(accu_var), + frame->comprehension_slots().Set(entry.accu_slot, std::move(accu_var), std::move(accu_var_attr)); frame->value_stack().Pop(1); } ComprehensionSlots::Slot* iter_slot = - frame->comprehension_slots().Get(iter_slot_); + frame->comprehension_slots().Get(entry.iter_slot); ABSL_DCHECK(iter_slot != nullptr); iter_slot->Set(); ComprehensionSlots::Slot* iter2_slot = - frame->comprehension_slots().Get(iter2_slot_); + frame->comprehension_slots().Get(entry.iter2_slot); ABSL_DCHECK(iter2_slot != nullptr); iter2_slot->Set(); - CEL_ASSIGN_OR_RETURN( - bool ok, - frame->iterator_stack().Peek()->Next2( - frame->descriptor_pool(), frame->message_factory(), frame->arena(), - iter_slot->mutable_value(), iter2_slot->mutable_value())); - if (!ok) { + absl::StatusOr ok = entry.iterator->Next2( + frame->descriptor_pool(), frame->message_factory(), frame->arena(), + iter_slot->mutable_value(), iter2_slot->mutable_value()); + if (!ok.ok()) { + frame->Abort(std::move(ok).status()); + return; + } + if (!*ok) { iter_slot->Clear(); iter2_slot->Clear(); - return frame->JumpTo(jump_offset_); + frame->JumpToOrAbort(jump_offset_); + return; + } + absl::Status inc_status = frame->IncrementIterations(); + if (!inc_status.ok()) { + frame->Abort(std::move(inc_status)); + return; } - CEL_RETURN_IF_ERROR(frame->IncrementIterations()); if (frame->enable_unknowns()) { *iter_slot->mutable_attribute() = *iter2_slot->mutable_attribute() = frame->value_stack().PeekAttribute().Step( @@ -594,12 +624,18 @@ absl::Status ComprehensionNextStep::Evaluate2(ExecutionFrame* frame) const { iter2_slot->attribute().attribute()); } } - return absl::OkStatus(); } -absl::Status ComprehensionCondStep::Evaluate1(ExecutionFrame* frame) const { +void ComprehensionCondStep::Evaluate1(ExecutionFrame* frame) const { + if (frame->iterator_stack().empty()) { + frame->Abort(absl::InternalError("Iterator stack underflow")); + return; + } + const IteratorStack::Entry& entry = *frame->iterator_stack().Peek(); if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + frame->Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; } const Value& top = frame->value_stack().Peek(); switch (top.kind()) { @@ -607,33 +643,45 @@ absl::Status ComprehensionCondStep::Evaluate1(ExecutionFrame* frame) const { break; case ValueKind::kError: ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: + case ValueKind::kUnknown: { frame->value_stack().SwapAndPop(2, 1); - frame->comprehension_slots().ClearSlot(iter_slot_); - frame->comprehension_slots().ClearSlot(accu_slot_); + frame->comprehension_slots().ClearSlot(entry.iter_slot); + frame->comprehension_slots().ClearSlot(entry.accu_slot); frame->iterator_stack().Pop(); - return frame->JumpTo(error_jump_offset_); - default: + frame->JumpToOrAbort(error_jump_offset_); + return; + } + default: { frame->value_stack().PopAndPush( 2, cel::ErrorValue::From( CreateNoMatchingOverloadError(""), frame->arena())); - frame->comprehension_slots().ClearSlot(iter_slot_); - frame->comprehension_slots().ClearSlot(accu_slot_); + frame->comprehension_slots().ClearSlot(entry.iter_slot); + frame->comprehension_slots().ClearSlot(entry.accu_slot); frame->iterator_stack().Pop(); - return frame->JumpTo(error_jump_offset_); + frame->JumpToOrAbort(error_jump_offset_); + return; + } } const bool loop_condition = absl::implicit_cast(top.GetBool()); + const bool short_circuiting = frame->options().short_circuiting; frame->value_stack().Pop(1); // loop_condition - if (!loop_condition && shortcircuiting_) { - return frame->JumpTo(jump_offset_); + if (!loop_condition && short_circuiting) { + frame->JumpToOrAbort(jump_offset_); + return; } - return absl::OkStatus(); } -absl::Status ComprehensionCondStep::Evaluate2(ExecutionFrame* frame) const { +void ComprehensionCondStep::Evaluate2(ExecutionFrame* frame) const { + if (frame->iterator_stack().empty()) { + frame->Abort(absl::InternalError("Iterator stack underflow")); + return; + } + const IteratorStack::Entry& entry = *frame->iterator_stack().Peek(); if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + frame->Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; } const Value& top = frame->value_stack().Peek(); switch (top.kind()) { @@ -641,30 +689,35 @@ absl::Status ComprehensionCondStep::Evaluate2(ExecutionFrame* frame) const { break; case ValueKind::kError: ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: + case ValueKind::kUnknown: { frame->value_stack().SwapAndPop(2, 1); - frame->comprehension_slots().ClearSlot(iter_slot_); - frame->comprehension_slots().ClearSlot(iter2_slot_); - frame->comprehension_slots().ClearSlot(accu_slot_); + frame->comprehension_slots().ClearSlot(entry.iter_slot); + frame->comprehension_slots().ClearSlot(entry.iter2_slot); + frame->comprehension_slots().ClearSlot(entry.accu_slot); frame->iterator_stack().Pop(); - return frame->JumpTo(error_jump_offset_); - default: + frame->JumpToOrAbort(error_jump_offset_); + return; + } + default: { frame->value_stack().PopAndPush( 2, cel::ErrorValue::From( CreateNoMatchingOverloadError(""), frame->arena())); - frame->comprehension_slots().ClearSlot(iter_slot_); - frame->comprehension_slots().ClearSlot(iter2_slot_); - frame->comprehension_slots().ClearSlot(accu_slot_); + frame->comprehension_slots().ClearSlot(entry.iter_slot); + frame->comprehension_slots().ClearSlot(entry.iter2_slot); + frame->comprehension_slots().ClearSlot(entry.accu_slot); frame->iterator_stack().Pop(); - return frame->JumpTo(error_jump_offset_); + frame->JumpToOrAbort(error_jump_offset_); + return; + } } const bool loop_condition = absl::implicit_cast(top.GetBool()); + const bool short_circuiting = frame->options().short_circuiting; frame->value_stack().Pop(1); // loop_condition - if (!loop_condition && shortcircuiting_) { - return frame->JumpTo(jump_offset_); + if (!loop_condition && short_circuiting) { + frame->JumpToOrAbort(jump_offset_); + return; } - return absl::OkStatus(); } std::unique_ptr CreateDirectComprehensionStep( @@ -681,9 +734,15 @@ std::unique_ptr CreateDirectComprehensionStep( shortcircuiting, expr_id); } -std::unique_ptr CreateComprehensionFinishStep( - size_t accu_slot) { - return std::make_unique(accu_slot); +void EvaluateComprehensionFinishStep(size_t accu_slot, ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(2)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + frame.value_stack().SwapAndPop(2, 1); + frame.comprehension_slots().ClearSlot(accu_slot); + frame.iterator_stack().Pop(); } } // namespace google::api::expr::runtime diff --git a/eval/eval/comprehension_step.h b/eval/eval/comprehension_step.h index f16b62e13..8ec7960db 100644 --- a/eval/eval/comprehension_step.h +++ b/eval/eval/comprehension_step.h @@ -8,8 +8,8 @@ #include "absl/status/status.h" #include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" +#include "eval/eval/iterator_stack.h" namespace google::api::expr::runtime { @@ -25,75 +25,63 @@ namespace google::api::expr::runtime { // 8: 1 -> 2 // 9: ComprehensionFinishStep 2 -> 1 -class ComprehensionInitStep final : public ExpressionStepBase { +class ComprehensionInitStep final : public ExpressionStepLogic { public: - ComprehensionInitStep() : ExpressionStepBase() {} + ComprehensionInitStep(size_t iter_slot, size_t iter2_slot, size_t accu_slot) + : iter_slot_(iter_slot), + iter2_slot_(iter2_slot), + accu_slot_(accu_slot), + has_iter2_(true) {} + ComprehensionInitStep(size_t iter_slot, size_t accu_slot) + : iter_slot_(iter_slot), + iter2_slot_(0), + accu_slot_(accu_slot), + has_iter2_(false) {} void set_error_jump_offset(int offset) { error_jump_offset_ = offset; } absl::Status Evaluate(ExecutionFrame* frame) const override; private: + const size_t iter_slot_; + const size_t iter2_slot_; + const size_t accu_slot_; + bool has_iter2_ = false; int error_jump_offset_ = std::numeric_limits::max(); }; -class ComprehensionNextStep final : public ExpressionStepBase { +class ComprehensionNextStep final { public: - ComprehensionNextStep(size_t iter_slot, size_t iter2_slot, size_t accu_slot) - : ExpressionStepBase(), - iter_slot_(iter_slot), - iter2_slot_(iter2_slot), - accu_slot_(accu_slot) {} + ComprehensionNextStep() = default; void set_jump_offset(int offset) { jump_offset_ = offset; } void set_error_jump_offset(int offset) { error_jump_offset_ = offset; } - absl::Status Evaluate(ExecutionFrame* frame) const override { - return iter_slot_ == iter2_slot_ ? Evaluate1(frame) : Evaluate2(frame); - } - - private: - absl::Status Evaluate1(ExecutionFrame* frame) const; + void Evaluate1(ExecutionFrame* frame) const; - absl::Status Evaluate2(ExecutionFrame* frame) const; + void Evaluate2(ExecutionFrame* frame) const; - const size_t iter_slot_; - const size_t iter2_slot_; - const size_t accu_slot_; - int jump_offset_ = std::numeric_limits::max(); - int error_jump_offset_ = std::numeric_limits::max(); + private: + int32_t jump_offset_ = std::numeric_limits::max(); + int32_t error_jump_offset_ = std::numeric_limits::max(); }; -class ComprehensionCondStep final : public ExpressionStepBase { +class ComprehensionCondStep final { public: - ComprehensionCondStep(size_t iter_slot, size_t iter2_slot, size_t accu_slot, - bool shortcircuiting) - : ExpressionStepBase(), - iter_slot_(iter_slot), - iter2_slot_(iter2_slot), - accu_slot_(accu_slot), - shortcircuiting_(shortcircuiting) {} + ComprehensionCondStep() = default; void set_jump_offset(int offset) { jump_offset_ = offset; } void set_error_jump_offset(int offset) { error_jump_offset_ = offset; } - absl::Status Evaluate(ExecutionFrame* frame) const override { - return iter_slot_ == iter2_slot_ ? Evaluate1(frame) : Evaluate2(frame); - } - - private: - absl::Status Evaluate1(ExecutionFrame* frame) const; + void Evaluate1(ExecutionFrame* frame) const; - absl::Status Evaluate2(ExecutionFrame* frame) const; + void Evaluate2(ExecutionFrame* frame) const; - const size_t iter_slot_; - const size_t iter2_slot_; - const size_t accu_slot_; - int jump_offset_ = std::numeric_limits::max(); - int error_jump_offset_ = std::numeric_limits::max(); - const bool shortcircuiting_; + private: + int32_t jump_offset_ = std::numeric_limits::max(); + int32_t error_jump_offset_ = std::numeric_limits::max(); }; // Creates a step for executing a comprehension. @@ -106,11 +94,10 @@ std::unique_ptr CreateDirectComprehensionStep( std::unique_ptr result_step, bool shortcircuiting, int64_t expr_id); -// Creates a cleanup step for the comprehension. +// Runs a cleanup step for the comprehension. // Removes the comprehension context then pushes the 'result' sub expression to // the top of the stack. -std::unique_ptr CreateComprehensionFinishStep( - size_t accu_slot); +void EvaluateComprehensionFinishStep(size_t accu_slot, ExecutionFrame& frame); } // namespace google::api::expr::runtime diff --git a/eval/eval/comprehension_step_test.cc b/eval/eval/comprehension_step_test.cc index ef01b034d..ee8b764c1 100644 --- a/eval/eval/comprehension_step_test.cc +++ b/eval/eval/comprehension_step_test.cc @@ -7,7 +7,6 @@ #include "cel/expr/syntax.pb.h" #include "google/protobuf/struct.pb.h" -#include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -44,7 +43,6 @@ using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::BoolValue; using ::cel::Expr; -using ::cel::IdentExpr; using ::cel::IntValue; using ::cel::TypeProvider; using ::cel::Value; @@ -57,12 +55,6 @@ using ::testing::Eq; using ::testing::Return; using ::testing::SizeIs; -IdentExpr CreateIdent(const std::string& var) { - IdentExpr expr; - expr.set_name(var); - return expr; -} - class ListKeysStepTest : public testing::Test { public: ListKeysStepTest() = default; @@ -104,7 +96,8 @@ MATCHER_P(CelStringValue, val, "") { TEST_F(ListKeysStepTest, MapPartiallyUnknown) { ExecutionPath path; path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); - auto init_step = std::make_unique(); + auto init_step = + std::make_unique(/*iter_slot=*/0, /*accu_slot=*/0); init_step->set_error_jump_offset(1); path.push_back(ExpressionStep::MakeGenericStep(std::move(init_step))); path.push_back(ExpressionStep::MakeGenericStep( @@ -141,7 +134,8 @@ TEST_F(ListKeysStepTest, MapPartiallyUnknown) { TEST_F(ListKeysStepTest, ErrorPassedThrough) { ExecutionPath path; path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); - auto init_step = std::make_unique(); + auto init_step = + std::make_unique(/*iter_slot=*/0, /*accu_slot=*/0); init_step->set_error_jump_offset(1); path.push_back(ExpressionStep::MakeGenericStep(std::move(init_step))); path.push_back(ExpressionStep::MakeGenericStep( @@ -165,7 +159,8 @@ TEST_F(ListKeysStepTest, ErrorPassedThrough) { TEST_F(ListKeysStepTest, UnknownSetPassedThrough) { ExecutionPath path; path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); - auto init_step = std::make_unique(); + auto init_step = + std::make_unique(/*iter_slot=*/0, /*accu_slot=*/0); init_step->set_error_jump_offset(1); path.push_back(ExpressionStep::MakeGenericStep(std::move(init_step))); path.push_back(ExpressionStep::MakeGenericStep( diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 86ae84d3b..308605809 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -30,6 +30,8 @@ #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" +#include "eval/eval/comprehension_slots.h" +#include "eval/eval/comprehension_step.h" #include "eval/eval/lazy_init_step.h" #include "eval/eval/logic_step.h" #include "internal/status_macros.h" @@ -108,6 +110,17 @@ class EvaluationStatus final { alignas(absl::Status) char status_[sizeof(absl::Status)]; }; +void EvaluateReadSlotStep(size_t slot_index, ExecutionFrame& frame) { + const ComprehensionSlots::Slot* slot = + frame.comprehension_slots().Get(slot_index); + if (!slot->Has()) { + frame.Abort(absl::InternalError(absl::StrCat( + "Comprehension variable read out of scope: ", slot_index))); + return; + } + frame.value_stack().Push(slot->value(), slot->attribute()); +} + } // namespace void ExpressionStep::Evaluate(ExecutionFrame* context) const { @@ -158,11 +171,31 @@ void ExpressionStep::Evaluate(ExecutionFrame* context) const { case ExpressionStepKind::kBooleanAnd: EvaluateBoolLogicStep(BoolLogicKind::kAnd, u_.arg_count, *context); break; + case ExpressionStepKind::kComprehensionFinish: + EvaluateComprehensionFinishStep(u_.slot_index, *context); + break; + case ExpressionStepKind::kComprehensionNext: + u_.next_step.Evaluate1(context); + break; + case ExpressionStepKind::kComprehensionNext2: + u_.next_step.Evaluate2(context); + break; + case ExpressionStepKind::kComprehensionCond: + u_.cond_step.Evaluate1(context); + break; + case ExpressionStepKind::kComprehensionCond2: + u_.cond_step.Evaluate2(context); + break; + case ExpressionStepKind::kReadSlot: + EvaluateReadSlotStep(u_.slot_index, *context); + break; case ExpressionStepKind::kMovedFrom: - default: context->Abort( absl::InternalError("ExpressionStep::Evaluate called on moved-from " "object")); + break; + default: + ABSL_UNREACHABLE(); } } @@ -313,6 +346,22 @@ bool IsConstant(const ExpressionStep& step) { } } +ComprehensionCondStep* GetIfComprehensionCondStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kComprehensionCond || + step.header_.kind == ExpressionStepKind::kComprehensionCond2) { + return &step.u_.cond_step; + } + return nullptr; +} + +ComprehensionNextStep* GetIfComprehensionNextStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kComprehensionNext || + step.header_.kind == ExpressionStepKind::kComprehensionNext2) { + return &step.u_.next_step; + } + return nullptr; +} + absl::Status WrappedDirectStep::Evaluate(ExecutionFrame* frame) const { cel::Value result; AttributeTrail attribute_trail; diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 73772845c..8463f4ad6 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -35,8 +35,10 @@ #include "common/value.h" #include "eval/eval/attribute_utility.h" #include "eval/eval/comprehension_slots.h" +#include "eval/eval/comprehension_step.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_stack.h" +#include "eval/eval/expression_step_logic.h" #include "eval/eval/iterator_stack.h" #include "eval/eval/lazy_init_step.h" #include "eval/eval/logic_step.h" @@ -59,8 +61,6 @@ class ExecutionFrame; using EvaluationListener = cel::TraceableProgram::EvaluationListener; -class ExpressionStepLogic; - enum class ExpressionStepKind : uint16_t { kMovedFrom = 0, kGenericLogic = 1, @@ -78,6 +78,15 @@ enum class ExpressionStepKind : uint16_t { kNotStrictlyFalse = 12, kBooleanOr = 13, kBooleanAnd = 14, + // Comprehension steps. + // Init step doesn't fit inline and is slow anyway, so just use a generic + // step. + kComprehensionFinish = 15, + kComprehensionNext = 16, + kComprehensionCond = 17, + kComprehensionNext2 = 18, + kComprehensionCond2 = 19, + kReadSlot = 20, }; class ExpressionStep { @@ -170,6 +179,44 @@ class ExpressionStep { return step; } + static ExpressionStep MakeComprehensionFinishStep(size_t accu_slot, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionFinish, id); + step.u_.slot_index = accu_slot; + return step; + } + + static ExpressionStep MakeComprehensionNextStep(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionNext, id); + step.u_.next_step = ComprehensionNextStep(); + return step; + } + + static ExpressionStep MakeComprehensionNext2Step(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionNext2, id); + step.u_.next_step = ComprehensionNextStep(); + return step; + } + + static ExpressionStep MakeComprehensionCondStep(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionCond, id); + step.u_.cond_step = ComprehensionCondStep(); + return step; + } + + static ExpressionStep MakeComprehensionCond2Step(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kComprehensionCond2, id); + step.u_.cond_step = ComprehensionCondStep(); + return step; + } + + static ExpressionStep MakeReadSlotStep(size_t slot_index, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kReadSlot, id); + ABSL_DCHECK_LT(slot_index, std::numeric_limits::max()); + step.u_.slot_index = slot_index; + return step; + } + private: struct Header { ExpressionStepKind kind; @@ -201,6 +248,10 @@ class ExpressionStep { friend bool GetIfConstant(const ExpressionStep& step, cel::Value& out); friend bool IsConstant(const ExpressionStep& step); + friend ComprehensionCondStep* GetIfComprehensionCondStep( + ExpressionStep& step); + friend ComprehensionNextStep* GetIfComprehensionNextStep( + ExpressionStep& step); Header header_; // Note: ptr members are 'owned' by the step. @@ -217,6 +268,8 @@ class ExpressionStep { size_t slot_index; ClearSlotStepInfo clear_slots; size_t arg_count; + ComprehensionCondStep cond_step; + ComprehensionNextStep next_step; Data() : empty(nullptr) {} ~Data() {} @@ -232,28 +285,6 @@ class ExpressionStep { static_assert(sizeof(ExpressionStep) == 16); #endif -// Class Expression represents single execution step. -class ExpressionStepLogic { - public: - virtual ~ExpressionStepLogic() = default; - - // Performs actual evaluation. - // Values are passed between Expression objects via EvaluatorStack, which is - // supplied with context. - // Also, Expression gets values supplied by caller though Activation - // interface. - // ExpressionStep instances can in specific cases - // modify execution order(perform jumps). - virtual absl::Status Evaluate(ExecutionFrame* context) const = 0; - - // Return the type of the underlying expression step for special handling in - // the planning phase. This should only be overridden by special cases, and - // callers must not make any assumptions about the default case. - virtual cel::NativeTypeId GetNativeTypeId() const { - return cel::NativeTypeId(); - } -}; - // Wrapper for direct steps to work with the stack machine impl. class WrappedDirectStep : public ExpressionStepLogic { public: @@ -575,6 +606,21 @@ class ExecutionFrame : public ExecutionFrameBase { return absl::OkStatus(); } + void JumpToOrAbort(int offset) { + ABSL_DCHECK_LE(offset, static_cast(execution_path_.size())); + ABSL_DCHECK_GE(offset, -static_cast(pc_)); + + int new_pc = static_cast(pc_) + offset; + if (new_pc < 0 || new_pc > static_cast(execution_path_.size())) { + Abort(absl::Status(absl::StatusCode::kInternal, + absl::StrCat("Jump address out of range: position: ", + pc_, ", offset: ", offset, + ", range: ", execution_path_.size()))); + return; + } + pc_ = static_cast(new_pc); + } + // Move pc to a subexpression. // // Unlike a `Call` in a programming language, the subexpression is evaluated @@ -740,6 +786,9 @@ bool GetIfConstant(const ExpressionStep& step, cel::Value& out); // Checks if the step is a constant. bool IsConstant(const ExpressionStep& step); +ComprehensionCondStep* GetIfComprehensionCondStep(ExpressionStep& step); +ComprehensionNextStep* GetIfComprehensionNextStep(ExpressionStep& step); + // Implementation details. inline ExpressionStep::~ExpressionStep() { @@ -763,6 +812,12 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kNotStrictlyFalse: case ExpressionStepKind::kBooleanOr: case ExpressionStepKind::kBooleanAnd: + case ExpressionStepKind::kComprehensionFinish: + case ExpressionStepKind::kComprehensionCond: + case ExpressionStepKind::kComprehensionNext: + case ExpressionStepKind::kComprehensionCond2: + case ExpressionStepKind::kComprehensionNext2: + case ExpressionStepKind::kReadSlot: break; default: ABSL_UNREACHABLE(); diff --git a/eval/eval/expression_step_base.h b/eval/eval/expression_step_base.h index 565dc5fa5..7f4425098 100644 --- a/eval/eval/expression_step_base.h +++ b/eval/eval/expression_step_base.h @@ -3,7 +3,7 @@ #include -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { diff --git a/eval/eval/expression_step_logic.h b/eval/eval/expression_step_logic.h new file mode 100644 index 000000000..2d256636f --- /dev/null +++ b/eval/eval/expression_step_logic.h @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_LOGIC_H_ +#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_LOGIC_H_ + +#include "absl/status/status.h" +#include "common/native_type.h" + +namespace google::api::expr::runtime { + +class ExecutionFrame; + +// ExpressionStepLogic is the base class for generic expression steps that are +// not implemented directly in the evaluator core. +class ExpressionStepLogic { + public: + virtual ~ExpressionStepLogic() = default; + + // Performs actual evaluation. + // Values are passed between Expression objects via EvaluatorStack, which is + // supplied with context. + // Also, Expression gets values supplied by caller though Activation + // interface. + // ExpressionStep instances can in specific cases + // modify execution order(perform jumps). + virtual absl::Status Evaluate(ExecutionFrame* context) const = 0; + + // Return the type of the underlying expression step for special handling in + // the planning phase. This should only be overridden by special cases, and + // callers must not make any assumptions about the default case. + virtual cel::NativeTypeId GetNativeTypeId() const { + return cel::NativeTypeId(); + } +}; + +} // namespace google::api::expr::runtime + +#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_LOGIC_H_ diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index 9d6edef72..eee511a7e 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -17,6 +17,7 @@ #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" #include "internal/status_macros.h" @@ -94,24 +95,6 @@ absl::StatusOr LookupSlot( return slot; } -class SlotStep : public ExpressionStepBase { - public: - SlotStep(absl::string_view name, size_t slot_index) - : name_(name), slot_index_(slot_index) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - CEL_ASSIGN_OR_RETURN(const ComprehensionSlots::Slot* slot, - LookupSlot(name_, slot_index_, *frame)); - frame->value_stack().Push(slot->value(), slot->attribute()); - return absl::OkStatus(); - } - - private: - std::string name_; - - size_t slot_index_; -}; - class DirectIdentStep : public DirectExpressionStep { public: DirectIdentStep(absl::string_view name, int64_t expr_id) @@ -167,9 +150,4 @@ std::unique_ptr CreateIdentStep( return std::make_unique(name); } -std::unique_ptr CreateIdentStepForSlot( - const absl::string_view name, size_t slot_index) { - return std::make_unique(name, slot_index); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step.h b/eval/eval/ident_step.h index ad65478bd..508056827 100644 --- a/eval/eval/ident_step.h +++ b/eval/eval/ident_step.h @@ -5,10 +5,9 @@ #include #include -#include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { @@ -21,10 +20,6 @@ std::unique_ptr CreateDirectSlotIdentStep( // Factory method for Ident - based Execution step std::unique_ptr CreateIdentStep(absl::string_view name); -// Factory method for identifier that has been assigned to a slot. -std::unique_ptr CreateIdentStepForSlot( - absl::string_view name, size_t slot_index); - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_IDENT_STEP_H_ diff --git a/eval/eval/iterator_stack.h b/eval/eval/iterator_stack.h index 9b5daa889..0b0de14aa 100644 --- a/eval/eval/iterator_stack.h +++ b/eval/eval/iterator_stack.h @@ -27,8 +27,15 @@ namespace cel::runtime_internal { class IteratorStack final { public: + struct Entry { + absl_nonnull ValueIteratorPtr iterator; + size_t iter_slot; + size_t iter2_slot; + size_t accu_slot; + }; + explicit IteratorStack(size_t max_size) : max_size_(max_size) { - iterators_.reserve(max_size_); + entries_.reserve(max_size_); } IteratorStack(const IteratorStack&) = delete; @@ -37,38 +44,55 @@ class IteratorStack final { IteratorStack& operator=(const IteratorStack&) = delete; IteratorStack& operator=(IteratorStack&&) = delete; - size_t size() const { return iterators_.size(); } + size_t size() const { return entries_.size(); } - bool empty() const { return iterators_.empty(); } + bool empty() const { return entries_.empty(); } - bool full() const { return iterators_.size() == max_size_; } + bool full() const { return entries_.size() == max_size_; } size_t max_size() const { return max_size_; } - void Clear() { iterators_.clear(); } + void Clear() { entries_.clear(); } + + void Push(absl_nonnull ValueIteratorPtr iterator, size_t iter_slot, + size_t iter2_slot, size_t accu_slot) { + ABSL_DCHECK(!full()); + ABSL_DCHECK(iterator != nullptr); + + entries_.push_back( + Entry{std::move(iterator), iter_slot, iter2_slot, accu_slot}); + } - void Push(absl_nonnull ValueIteratorPtr iterator) { + void Push(absl_nonnull ValueIteratorPtr iterator, size_t iter_slot, + size_t accu_slot) { ABSL_DCHECK(!full()); ABSL_DCHECK(iterator != nullptr); - iterators_.push_back(std::move(iterator)); + entries_.push_back(Entry{std::move(iterator), iter_slot, 0, accu_slot}); + } + + ValueIterator* absl_nonnull PeekIterator() { + ABSL_DCHECK(!empty()); + + return entries_.back().iterator.get(); } - ValueIterator* absl_nonnull Peek() { + // Returns a pointer to the top entry in the stack. + // Invalidated by Pop() and Push(). + Entry* absl_nonnull Peek() { ABSL_DCHECK(!empty()); - ABSL_DCHECK(iterators_.back() != nullptr); - return iterators_.back().get(); + return &entries_.back(); } void Pop() { ABSL_DCHECK(!empty()); - iterators_.pop_back(); + entries_.pop_back(); } private: - std::vector iterators_; + std::vector entries_; size_t max_size_; }; From e1cbba680fa880789960f2dffedf7b1960570565 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 1 Oct 2026 09:09:41 -0700 Subject: [PATCH 251/350] Add instructions for basic jump steps (implementing ?:, &&, || shortcircuiting) PiperOrigin-RevId: 991716020 --- eval/compiler/BUILD | 2 +- eval/compiler/flat_expr_builder.cc | 216 +++++++++++++++++------------ eval/eval/BUILD | 24 +--- eval/eval/evaluator_core.cc | 85 ++++++++++++ eval/eval/evaluator_core.h | 84 +++++++++++ eval/eval/evaluator_core_test.cc | 84 +++++++++++ eval/eval/jump_step.cc | 171 ----------------------- eval/eval/jump_step.h | 74 ---------- eval/eval/optional_or_step.cc | 87 +++++------- eval/eval/optional_or_step.h | 21 ++- 10 files changed, 431 insertions(+), 417 deletions(-) delete mode 100644 eval/eval/jump_step.cc delete mode 100644 eval/eval/jump_step.h diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 541eb4d79..8df9912bf 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -123,7 +123,6 @@ cc_library( "//eval/eval:expression_step_logic", "//eval/eval:function_step", "//eval/eval:ident_step", - "//eval/eval:jump_step", "//eval/eval:optional_or_step", "//eval/eval:select_step", "//eval/eval:shadowable_value_step", @@ -138,6 +137,7 @@ cc_library( "//runtime/internal:issue_collector", "//runtime/internal:runtime_env", "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 514e28df4..c84820d35 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -75,7 +75,6 @@ #include "eval/eval/expression_step_logic.h" #include "eval/eval/function_step.h" #include "eval/eval/ident_step.h" -#include "eval/eval/jump_step.h" #include "eval/eval/lazy_init_step.h" #include "eval/eval/logic_step.h" #include "eval/eval/optional_or_step.h" @@ -157,13 +156,6 @@ struct ProgramStepIndex { // A convenience wrapper for offset-calculating logic. class Jump { public: - // Default constructor for empty jump. - // - // Users must check that jump is non-empty before calling member functions. - explicit Jump() : self_index_{-1, nullptr}, jump_step_(nullptr) {} - Jump(ProgramStepIndex self_index, JumpStepBase* jump_step) - : self_index_(self_index), jump_step_(jump_step) {} - static absl::StatusOr CalculateOffset(ProgramStepIndex base, ProgramStepIndex target) { if (target.subexpression != base.subexpression) { @@ -173,21 +165,19 @@ class Jump { } int offset = base.subexpression->CalculateOffset(base.index, target.index); - return offset; - } - absl::Status set_target(ProgramStepIndex target) { - CEL_ASSIGN_OR_RETURN(int offset, CalculateOffset(self_index_, target)); + // The offset may be packed into a 31-bit int field. Technically, the offset + // could be larger for a well-formed expression with more than 2**30 steps, + // but such a case is unlikely to exist at this time. + constexpr int kMaxJumpOffset = (1 << 30) - 1; + constexpr int kMinJumpOffset = -(1 << 30); - jump_step_->set_jump_offset(offset); - return absl::OkStatus(); + if (offset < kMinJumpOffset || offset > kMaxJumpOffset) { + return absl::InternalError("Jump offset exceeds supported range: " + + absl::StrCat(offset)); + } + return offset; } - - bool exists() { return jump_step_ != nullptr; } - - private: - ProgramStepIndex self_index_; - JumpStepBase* jump_step_; }; class CondVisitor { @@ -225,7 +215,7 @@ class LogicalCondVisitor : public CondVisitor { private: FlatExprVisitor* visitor_; const bool is_or_; - std::vector jump_steps_; + std::vector jump_steps_; bool short_circuiting_; }; @@ -247,7 +237,8 @@ class OptionalOrCondVisitor : public CondVisitor { private: FlatExprVisitor* visitor_; const bool is_or_value_; - std::vector jump_steps_; + OptionalHasValueJumpStep* jump_step_ = nullptr; + ProgramStepIndex jump_step_pos_{-1, nullptr}; bool short_circuiting_; }; @@ -261,9 +252,8 @@ class TernaryCondVisitor : public CondVisitor { private: FlatExprVisitor* visitor_; - Jump jump_to_second_; - Jump error_jump_; - Jump jump_after_first_; + std::optional cond_jump_pos_; + std::optional jump_after_first_pos_; }; class ExhaustiveTernaryCondVisitor : public CondVisitor { @@ -1583,6 +1573,7 @@ class FlatExprVisitor : public cel::AstVisitor { } if (GetOptimizableListAppendOperand(comprehension.comprehension) == &expr) { + // Avoid wrapping the appended element in a create list step. return; } } @@ -2262,23 +2253,30 @@ void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { ExpressionStep::MakeBooleanAndStep(num_args, expr->id())); } if (short_circuiting_ && !jump_steps_.empty()) { - for (auto& jump : jump_steps_) { - visitor_->SetProgressStatusIfError( - jump.set_target(visitor_->GetCurrentIndex())); + ProgramStepIndex target = visitor_->GetCurrentIndex(); + for (const ProgramStepIndex& jump_pos : jump_steps_) { + absl::StatusOr offset = Jump::CalculateOffset(jump_pos, target); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + continue; + } + ExpressionStep* step = + jump_pos.subexpression->GetIfExpressionStep(jump_pos.index); + BoolJumpStepInfo* jump_info = GetIfBoolJumpStep(step); + if (jump_info != nullptr) { + jump_info->set = true; + jump_info->offset = *offset; + } } } } if (short_circuiting_ && arg_num < last_arg_index) { - std::unique_ptr jump_step = - is_or_ - ? CreateCondJumpStep(true, {}, /*expected_stack_size=*/arg_num + 1) - : CreateCondJumpStep(false, {}, - /*expected_stack_size=*/arg_num + 1); ProgramStepIndex index = visitor_->GetCurrentIndex(); - if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); - jump_step_ptr) { - jump_steps_.push_back(Jump(index, jump_step_ptr)); - } + visitor_->AddStep(is_or_ ? ExpressionStep::MakeBooleanOrJumpStep( + /*arg_count=*/arg_num + 1) + : ExpressionStep::MakeBooleanAndJumpStep( + /*arg_count=*/arg_num + 1)); + jump_steps_.push_back(index); } } @@ -2298,20 +2296,14 @@ void OptionalOrCondVisitor::PostVisitTarget(const cel::Expr* expr) { if (visitor_->PlanRecursiveProgram()) { return; } - if (short_circuiting_) { - // If first branch evaluation result is enough to determine output, - // jump over the second branch and provide result of the first argument as - // final output. - // Retain a pointer to the jump step so we can update the target after - // planning the second argument. - std::unique_ptr jump_step = - CreateOptionalHasValueJumpStep(is_or_value_); - ProgramStepIndex index = visitor_->GetCurrentIndex(); - if (JumpStepBase* jump_step_ptr = visitor_->AddStep(std::move(jump_step)); - jump_step_ptr) { - jump_steps_.push_back(Jump(index, jump_step_ptr)); - } + if (!short_circuiting_) { + return; } + + // Keep a pointer to the jump step so we can update the target after + // planning the second argument. + jump_step_pos_ = visitor_->GetCurrentIndex(); + jump_step_ = visitor_->AddStep(CreateOptionalHasValueJumpStep(is_or_value_)); } void OptionalOrCondVisitor::PostVisit(const cel::Expr* expr) { @@ -2321,12 +2313,20 @@ void OptionalOrCondVisitor::PostVisit(const cel::Expr* expr) { } visitor_->AddStep(CreateOptionalOrStep(is_or_value_), expr->id()); - if (short_circuiting_) { - for (auto& jump : jump_steps_) { - visitor_->SetProgressStatusIfError( - jump.set_target(visitor_->GetCurrentIndex())); - } + if (!short_circuiting_) { + return; } + + ABSL_DCHECK(jump_step_ != nullptr) + << "OptionalOrCondVisitor::PostVisit: jump_step_ is null"; + absl::StatusOr offset = + Jump::CalculateOffset(jump_step_pos_, visitor_->GetCurrentIndex()); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + return; + } + + jump_step_->set_jump_offset(*offset); } void TernaryCondVisitor::PreVisit(const cel::Expr* expr) { @@ -2352,37 +2352,35 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { // condition argument for ternary operator if (arg_num == 0) { - // Jump in case of error or non-bool - ProgramStepIndex error_jump_pos = visitor_->GetCurrentIndex(); - auto* error_jump = visitor_->AddStep(CreateBoolCheckJumpStep()); - if (error_jump) { - error_jump_ = Jump(error_jump_pos, error_jump); - } - - // Jump to the second branch of execution - // Value is to be removed from the stack. - ProgramStepIndex cond_jump_pos = visitor_->GetCurrentIndex(); - auto* jump_to_second = visitor_->AddStep(CreateTernaryCondJumpStep()); - if (jump_to_second) { - jump_to_second_ = - Jump(cond_jump_pos, static_cast(jump_to_second)); - } + // Jump in case of error or non-bool, or jump to the second branch of + // execution if false. + cond_jump_pos_ = visitor_->GetCurrentIndex(); + visitor_->AddStep(ExpressionStep::MakeTernaryJumpStep()); } else if (arg_num == 1) { // Jump after the first and over the second branch of execution. - // Value is to be removed from the stack. - ProgramStepIndex jump_pos = visitor_->GetCurrentIndex(); - auto* jump_after_first = visitor_->AddStep(CreateJumpStep()); - if (!jump_after_first) { + jump_after_first_pos_ = visitor_->GetCurrentIndex(); + visitor_->AddStep(ExpressionStep::MakeFixedJumpStep()); + + ExpressionStep* cond_step = + cond_jump_pos_.has_value() + ? cond_jump_pos_->subexpression->GetIfExpressionStep( + cond_jump_pos_->index) + : nullptr; + TernaryJumpStepInfo* ternary_info = GetIfTernaryJumpStep(cond_step); + if (!visitor_->ValidateOrError( + ternary_info != nullptr, + "Error configuring ternary operator: jump_to_second_ is null")) { return; } - jump_after_first_ = Jump(jump_pos, jump_after_first); - if (visitor_->ValidateOrError( - jump_to_second_.exists(), - "Error configuring ternary operator: jump_to_second_ is null")) { - visitor_->SetProgressStatusIfError( - jump_to_second_.set_target(visitor_->GetCurrentIndex())); + absl::StatusOr offset = + Jump::CalculateOffset(*cond_jump_pos_, visitor_->GetCurrentIndex()); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + return; } + + ternary_info->jump_to_second_offset = *offset; } // Code executed after traversing the final branch of execution // (arg_num == 2) is placed in PostVisitCall, to make this method less @@ -2395,17 +2393,41 @@ void TernaryCondVisitor::PostVisit(const cel::Expr* expr) { return; } // Determine and set jump offset in jump instruction. + ExpressionStep* cond_step = + cond_jump_pos_.has_value() + ? cond_jump_pos_->subexpression->GetIfExpressionStep( + cond_jump_pos_->index) + : nullptr; + TernaryJumpStepInfo* ternary_info = GetIfTernaryJumpStep(cond_step); if (visitor_->ValidateOrError( - error_jump_.exists(), + ternary_info != nullptr, "Error configuring ternary operator: error_jump_ is null")) { - visitor_->SetProgressStatusIfError( - error_jump_.set_target(visitor_->GetCurrentIndex())); + absl::StatusOr offset = + Jump::CalculateOffset(*cond_jump_pos_, visitor_->GetCurrentIndex()); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + } else { + ternary_info->set = true; + ternary_info->error_offset = *offset; + } } + ExpressionStep* jump_after_first_step = + jump_after_first_pos_.has_value() + ? jump_after_first_pos_->subexpression->GetIfExpressionStep( + jump_after_first_pos_->index) + : nullptr; + FixedJumpStepInfo* fixed_info = GetIfFixedJumpStep(jump_after_first_step); if (visitor_->ValidateOrError( - jump_after_first_.exists(), + fixed_info != nullptr, "Error configuring ternary operator: jump_after_first_ is null")) { - visitor_->SetProgressStatusIfError( - jump_after_first_.set_target(visitor_->GetCurrentIndex())); + absl::StatusOr offset = Jump::CalculateOffset( + *jump_after_first_pos_, visitor_->GetCurrentIndex()); + if (!offset.ok()) { + visitor_->SetProgressStatusIfError(offset.status()); + } else { + fixed_info->set = true; + fixed_info->offset = *offset; + } } } @@ -2492,13 +2514,17 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( } case cel::LOOP_STEP: { ProgramStepIndex index = visitor_->GetCurrentIndex(); - auto* jump_to_next = visitor_->AddStep(CreateJumpStep()); - if (!jump_to_next) { + ExpressionStep* jump_step = + visitor_->AddStep(ExpressionStep::MakeFixedJumpStep()); + FixedJumpStepInfo* fixed_info = GetIfFixedJumpStep(jump_step); + if (fixed_info == nullptr) { + // either an error occurred earlier or planning is suppressed. break; } - Jump jump_helper(index, jump_to_next); - visitor_->SetProgressStatusIfError( - jump_helper.set_target(*next_step_pos_)); + CEL_ASSIGN_OR_RETURN(int offset, + Jump::CalculateOffset(index, *next_step_pos_)); + fixed_info->set = true; + fixed_info->offset = offset; // Set offsets jumping to the result step. if (auto* cond_step = GetCondStep(); cond_step != nullptr) { @@ -2506,6 +2532,8 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( Jump::CalculateOffset( *cond_step_pos_, visitor_->GetCurrentIndex())); cond_step->set_jump_offset(jump_from_cond); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); } if (auto* next_step = GetNextStep(); next_step != nullptr) { @@ -2514,6 +2542,8 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( *next_step_pos_, visitor_->GetCurrentIndex())); next_step->set_jump_offset(jump_from_next); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); } break; } @@ -2535,6 +2565,8 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( Jump::CalculateOffset( *next_step_pos_, visitor_->GetCurrentIndex())); next_step->set_error_jump_offset(jump_from_next); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); } if (auto* cond_step = GetCondStep(); cond_step != nullptr) { @@ -2542,6 +2574,8 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( Jump::CalculateOffset( *cond_step_pos_, visitor_->GetCurrentIndex())); cond_step->set_error_jump_offset(jump_from_cond); + } else { + return absl::InvalidArgumentError("Comprehension is malformed"); } break; } diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 5dfaf69a1..18374ab01 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -438,25 +438,6 @@ cc_library( ], ) -cc_library( - name = "jump_step", - srcs = [ - "jump_step.cc", - ], - hdrs = [ - "jump_step.h", - ], - deps = [ - ":evaluator_core", - ":expression_step_base", - "//common:value", - "//eval/internal:errors", - "@com_google_absl//absl/status", - "@com_google_absl//absl/types:optional", - "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", - ], -) - cc_library( name = "equality_steps", srcs = [ @@ -1195,15 +1176,14 @@ cc_library( ":direct_expression_step", ":evaluator_core", ":expression_step_base", - ":jump_step", + ":expression_step_logic", "//common:casting", + "//common:optional_ref", "//common:value", "//internal:status_macros", "//runtime/internal:errors", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_protobuf//:protobuf", ], diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 308605809..20357acec 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -20,6 +20,7 @@ #include #include +#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/log/absl_check.h" @@ -36,6 +37,7 @@ #include "eval/eval/logic_step.h" #include "internal/status_macros.h" #include "runtime/activation_interface.h" +#include "runtime/internal/errors.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -121,6 +123,53 @@ void EvaluateReadSlotStep(size_t slot_index, ExecutionFrame& frame) { frame.value_stack().Push(slot->value(), slot->attribute()); } +void EvaluateBoolJumpStep(const BoolJumpStepInfo& step, bool target, + ExecutionFrame& frame) { + ABSL_DCHECK(step.set) << "BoolJumpStep did not have a value set."; + if (!frame.value_stack().HasEnough(step.arg_count)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + const cel::Value& value = frame.value_stack().Peek(); + if (value.IsBool() && value.GetBool().NativeValue() == target) { + frame.value_stack().SwapAndPop(step.arg_count, step.arg_count - 1); + frame.JumpToOrAbort(step.offset); + } + // No-op if the value is not a bool or the value is not the target. + // Cleanup will happen if we hit a later jump or we fall-through. +} + +void EvaluateTernaryJumpStep(const TernaryJumpStepInfo& step, + ExecutionFrame& frame) { + ABSL_DCHECK(step.set) << "TernaryJumpStep did not have a value set."; + if (!frame.value_stack().HasEnough(1)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + const cel::Value& condition = frame.value_stack().Peek(); + switch (condition.kind()) { + case cel::ValueKind::kBool: + if (!condition.GetBool().NativeValue()) { + frame.JumpToOrAbort(step.jump_to_second_offset); + } + frame.value_stack().Pop(); + break; + default: + frame.value_stack().PopAndPush(cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + ""), + frame.arena())); + ABSL_FALLTHROUGH_INTENDED; + case cel::ValueKind::kError: + case cel::ValueKind::kUnknown: + // Propagate the error or unknown set. + frame.JumpToOrAbort(step.error_offset); + break; + } +} + } // namespace void ExpressionStep::Evaluate(ExecutionFrame* context) const { @@ -189,6 +238,20 @@ void ExpressionStep::Evaluate(ExecutionFrame* context) const { case ExpressionStepKind::kReadSlot: EvaluateReadSlotStep(u_.slot_index, *context); break; + case ExpressionStepKind::kBooleanOrJump: + EvaluateBoolJumpStep(u_.bool_jump_step, true, *context); + break; + case ExpressionStepKind::kBooleanAndJump: + EvaluateBoolJumpStep(u_.bool_jump_step, false, *context); + break; + case ExpressionStepKind::kTernaryJump: + EvaluateTernaryJumpStep(u_.ternary_jump_step, *context); + break; + case ExpressionStepKind::kFixedJump: + ABSL_DCHECK(u_.fixed_jump_step.set) + << "FixedJumpStep did not have a value set."; + context->JumpToOrAbort(u_.fixed_jump_step.offset); + break; case ExpressionStepKind::kMovedFrom: context->Abort( absl::InternalError("ExpressionStep::Evaluate called on moved-from " @@ -362,6 +425,28 @@ ComprehensionNextStep* GetIfComprehensionNextStep(ExpressionStep& step) { return nullptr; } +BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kBooleanOrJump || + step.header_.kind == ExpressionStepKind::kBooleanAndJump) { + return &step.u_.bool_jump_step; + } + return nullptr; +} + +TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kTernaryJump) { + return &step.u_.ternary_jump_step; + } + return nullptr; +} + +FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep& step) { + if (step.header_.kind == ExpressionStepKind::kFixedJump) { + return &step.u_.fixed_jump_step; + } + return nullptr; +} + absl::Status WrappedDirectStep::Evaluate(ExecutionFrame* frame) const { cel::Value result; AttributeTrail attribute_trail; diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 8463f4ad6..31ef7fdfd 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -87,6 +87,28 @@ enum class ExpressionStepKind : uint16_t { kComprehensionNext2 = 18, kComprehensionCond2 = 19, kReadSlot = 20, + kBooleanOrJump = 21, + kBooleanAndJump = 22, + kTernaryJump = 23, + kFixedJump = 24, +}; + +struct BoolJumpStepInfo { + size_t arg_count : 32; + bool set : 1; + int offset : 31; +}; + +struct TernaryJumpStepInfo { + bool set : 1; + int error_offset : 31; + int jump_to_second_offset : 32; +}; + +struct FixedJumpStepInfo { + bool set : 1; + int32_t reserved : 31; + int offset : 32; }; class ExpressionStep { @@ -217,6 +239,34 @@ class ExpressionStep { return step; } + static ExpressionStep MakeBooleanOrJumpStep(size_t arg_count, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBooleanOrJump, id); + ABSL_DCHECK_LT(arg_count, std::numeric_limits::max()); + step.u_.bool_jump_step = BoolJumpStepInfo{arg_count, false, 0}; + return step; + } + + static ExpressionStep MakeBooleanAndJumpStep(size_t arg_count, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kBooleanAndJump, id); + ABSL_DCHECK_LT(arg_count, std::numeric_limits::max()); + step.u_.bool_jump_step = BoolJumpStepInfo{arg_count, false, 0}; + return step; + } + + static ExpressionStep MakeTernaryJumpStep(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kTernaryJump, id); + step.u_.ternary_jump_step = TernaryJumpStepInfo{false, 0, 0}; + return step; + } + + static ExpressionStep MakeFixedJumpStep(int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kFixedJump, id); + step.u_.fixed_jump_step = FixedJumpStepInfo{false, 0, 0}; + return step; + } + private: struct Header { ExpressionStepKind kind; @@ -252,6 +302,9 @@ class ExpressionStep { ExpressionStep& step); friend ComprehensionNextStep* GetIfComprehensionNextStep( ExpressionStep& step); + friend BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep& step); + friend TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep& step); + friend FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep& step); Header header_; // Note: ptr members are 'owned' by the step. @@ -270,6 +323,9 @@ class ExpressionStep { size_t arg_count; ComprehensionCondStep cond_step; ComprehensionNextStep next_step; + BoolJumpStepInfo bool_jump_step; + TernaryJumpStepInfo ternary_jump_step; + FixedJumpStepInfo fixed_jump_step; Data() : empty(nullptr) {} ~Data() {} @@ -789,6 +845,30 @@ bool IsConstant(const ExpressionStep& step); ComprehensionCondStep* GetIfComprehensionCondStep(ExpressionStep& step); ComprehensionNextStep* GetIfComprehensionNextStep(ExpressionStep& step); +BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep& step); +inline BoolJumpStepInfo* GetIfBoolJumpStep(ExpressionStep* step) { + if (step == nullptr) { + return nullptr; + } + return GetIfBoolJumpStep(*step); +} + +TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep& step); +inline TernaryJumpStepInfo* GetIfTernaryJumpStep(ExpressionStep* step) { + if (step == nullptr) { + return nullptr; + } + return GetIfTernaryJumpStep(*step); +} + +FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep& step); +inline FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep* step) { + if (step == nullptr) { + return nullptr; + } + return GetIfFixedJumpStep(*step); +} + // Implementation details. inline ExpressionStep::~ExpressionStep() { @@ -818,6 +898,10 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kComprehensionCond2: case ExpressionStepKind::kComprehensionNext2: case ExpressionStepKind::kReadSlot: + case ExpressionStepKind::kBooleanOrJump: + case ExpressionStepKind::kBooleanAndJump: + case ExpressionStepKind::kTernaryJump: + case ExpressionStepKind::kFixedJump: break; default: ABSL_UNREACHABLE(); diff --git a/eval/eval/evaluator_core_test.cc b/eval/eval/evaluator_core_test.cc index 91d03b749..dde70857e 100644 --- a/eval/eval/evaluator_core_test.cc +++ b/eval/eval/evaluator_core_test.cc @@ -28,6 +28,7 @@ namespace google::api::expr::runtime { using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; using ::cel::IntValue; using ::cel::TypeProvider; using ::cel::interop_internal::CreateIntValue; @@ -37,6 +38,7 @@ using ::google::api::expr::runtime::RegisterBuiltinFunctions; using ::testing::_; using ::testing::ElementsAre; using ::testing::Eq; +using ::testing::HasSubstr; // Fake expression implementation // Pushes int64(0) on top of value stack. @@ -95,6 +97,88 @@ TEST(EvaluatorCoreTest, ExecutionFrameNext) { EXPECT_THAT(frame.Next(), Eq(nullptr)); } +TEST(EvaluatorCoreTest, JumpOutOfRangeAborts) { + ExecutionPath path; + google::protobuf::Arena arena; + cel::runtime_internal::RuntimeTypeProvider type_provider( + cel::internal::GetTestingDescriptorPool()); + + path.push_back(ExpressionStep::MakeFixedJumpStep()); + FixedJumpStepInfo* jump_info = GetIfFixedJumpStep(path.back()); + ASSERT_NE(jump_info, nullptr); + jump_info->set = true; + jump_info->offset = 1; + + cel::RuntimeOptions options; + cel::Activation activation; + FlatExpressionEvaluatorState state( + path.size(), + /*comprehension_slots_size=*/0, type_provider, + cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena); + ExecutionFrame frame(path, activation, options, state); + + EXPECT_THAT(frame.Evaluate(), + StatusIs(absl::StatusCode::kInternal, + HasSubstr("Jump address out of range"))); +} + +TEST(EvaluatorCoreTest, TestStepDataAccessors) { + ExpressionStep fixed_jump = ExpressionStep::MakeFixedJumpStep(); + EXPECT_NE(GetIfFixedJumpStep(fixed_jump), nullptr); + EXPECT_NE(GetIfFixedJumpStep(&fixed_jump), nullptr); + EXPECT_EQ(GetIfFixedJumpStep(static_cast(nullptr)), nullptr); + + ExpressionStep bool_or_jump = ExpressionStep::MakeBooleanOrJumpStep(2); + EXPECT_NE(GetIfBoolJumpStep(bool_or_jump), nullptr); + EXPECT_NE(GetIfBoolJumpStep(&bool_or_jump), nullptr); + EXPECT_EQ(GetIfBoolJumpStep(static_cast(nullptr)), nullptr); + + ExpressionStep bool_and_jump = ExpressionStep::MakeBooleanAndJumpStep(2); + EXPECT_NE(GetIfBoolJumpStep(bool_and_jump), nullptr); + EXPECT_NE(GetIfBoolJumpStep(&bool_and_jump), nullptr); + + ExpressionStep ternary_jump = ExpressionStep::MakeTernaryJumpStep(); + EXPECT_NE(GetIfTernaryJumpStep(ternary_jump), nullptr); + EXPECT_NE(GetIfTernaryJumpStep(&ternary_jump), nullptr); + EXPECT_EQ(GetIfTernaryJumpStep(static_cast(nullptr)), + nullptr); + + ExpressionStep comp_cond = ExpressionStep::MakeComprehensionCondStep(); + EXPECT_NE(GetIfComprehensionCondStep(comp_cond), nullptr); + + ExpressionStep comp_cond2 = ExpressionStep::MakeComprehensionCond2Step(); + EXPECT_NE(GetIfComprehensionCondStep(comp_cond2), nullptr); + + ExpressionStep comp_next = ExpressionStep::MakeComprehensionNextStep(); + EXPECT_NE(GetIfComprehensionNextStep(comp_next), nullptr); + + ExpressionStep comp_next2 = ExpressionStep::MakeComprehensionNext2Step(); + EXPECT_NE(GetIfComprehensionNextStep(comp_next2), nullptr); + + ExpressionStep const_step = ExpressionStep::MakeConstant(cel::IntValue(42)); + EXPECT_TRUE(IsConstant(const_step)); + cel::Value value; + EXPECT_TRUE(GetIfConstant(const_step, value)); + EXPECT_TRUE(value.IsInt()); + EXPECT_EQ(value.GetInt().NativeValue(), 42); + + ExpressionStep generic_step = ExpressionStep::MakeGenericStep( + std::make_unique()); + EXPECT_TRUE(generic_step.IsGenericStep()); + EXPECT_NE(generic_step.GetGenericStep(), nullptr); + + // Negative checks for mismatched step kinds. + EXPECT_EQ(GetIfFixedJumpStep(bool_or_jump), nullptr); + EXPECT_EQ(GetIfBoolJumpStep(fixed_jump), nullptr); + EXPECT_EQ(GetIfTernaryJumpStep(fixed_jump), nullptr); + EXPECT_EQ(GetIfComprehensionCondStep(fixed_jump), nullptr); + EXPECT_EQ(GetIfComprehensionNextStep(fixed_jump), nullptr); + EXPECT_FALSE(IsConstant(fixed_jump)); + EXPECT_FALSE(GetIfConstant(fixed_jump, value)); + EXPECT_FALSE(fixed_jump.IsGenericStep()); +} + TEST(EvaluatorCoreTest, SimpleEvaluatorTest) { ExecutionPath path; path.push_back(ExpressionStep::MakeGenericStep( diff --git a/eval/eval/jump_step.cc b/eval/eval/jump_step.cc deleted file mode 100644 index ff5d864f2..000000000 --- a/eval/eval/jump_step.cc +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2017 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "eval/eval/jump_step.h" - -#include -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/types/optional.h" -#include "common/value.h" -#include "eval/eval/evaluator_core.h" -#include "eval/internal/errors.h" - -namespace google::api::expr::runtime { - -namespace { - -using ::cel::BoolValue; -using ::cel::ErrorValue; -using ::cel::UnknownValue; -using ::cel::Value; -using ::cel::runtime_internal::CreateNoMatchingOverloadError; - -class JumpStep : public JumpStepBase { - public: - // Constructs FunctionStep that uses overloads specified. - explicit JumpStep(absl::optional jump_offset) - : JumpStepBase(jump_offset) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - return Jump(frame); - } -}; - -class CondJumpStep : public JumpStepBase { - public: - CondJumpStep(bool jump_condition, absl::optional jump_offset, - size_t stack_size) - : JumpStepBase(jump_offset), - jump_condition_(jump_condition), - stack_size_(stack_size) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - // Peek the top value - if (!frame->value_stack().HasEnough(stack_size_)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - const auto& value = frame->value_stack().Peek(); - const auto should_jump = - value.IsBool() && jump_condition_ == value.GetBool().NativeValue(); - - if (should_jump) { - frame->value_stack().SwapAndPop(stack_size_, stack_size_ - 1); - return Jump(frame); - } - - return absl::OkStatus(); - } - - private: - const bool jump_condition_; - const size_t stack_size_; -}; - -class TernaryCondJumpStep : public JumpStepBase { - public: - explicit TernaryCondJumpStep(absl::optional jump_offset) - : JumpStepBase(jump_offset) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - // Peek the top value - if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - const auto& value = frame->value_stack().Peek(); - const auto should_jump = value.IsBool() && !value.GetBool().NativeValue(); - - frame->value_stack().Pop(1); - - if (should_jump) { - return Jump(frame); - } - - return absl::OkStatus(); - } -}; - -class BoolCheckJumpStep : public JumpStepBase { - public: - // Checks if the top value is a boolean: - // - no-op if it is a boolean - // - jump to the label if it is an error value - // - jump to the label if it is unknown value - // - jump to the label if it is neither an error nor a boolean, pops it and - // pushes "no matching overload" error - explicit BoolCheckJumpStep(absl::optional jump_offset) - : JumpStepBase(jump_offset) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - // Peek the top value - if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - const Value& value = frame->value_stack().Peek(); - - if (value->Is()) { - return absl::OkStatus(); - } - - if (value->Is() || value->Is()) { - return Jump(frame); - } - - // Neither bool, error, nor unknown set. - Value error_value = cel::ErrorValue::From( - CreateNoMatchingOverloadError(""), frame->arena()); - - frame->value_stack().PopAndPush(std::move(error_value)); - return Jump(frame); - - return absl::OkStatus(); - } -}; - -} // namespace - -// Factory method for Conditional Jump step. -std::unique_ptr CreateCondJumpStep( - bool jump_condition, absl::optional jump_offset, - size_t expected_stack_size) { - return std::make_unique(jump_condition, jump_offset, - expected_stack_size); -} - -// Factory method for Ternary Conditional Jump step. -std::unique_ptr CreateTernaryCondJumpStep( - absl::optional jump_offset) { - return std::make_unique(jump_offset); -} - -// Factory method for Jump step. -std::unique_ptr CreateJumpStep(absl::optional jump_offset) { - return std::make_unique(jump_offset); -} - -// Factory method for Conditional Jump step. -// Conditional Jump requires a value to sit on the stack. -// If this value is an error or unknown, a jump is performed. -std::unique_ptr CreateBoolCheckJumpStep( - absl::optional jump_offset) { - return std::make_unique(jump_offset); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/jump_step.h b/eval/eval/jump_step.h deleted file mode 100644 index 5cef97329..000000000 --- a/eval/eval/jump_step.h +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_JUMP_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_JUMP_STEP_H_ - -#include -#include - -#include "cel/expr/syntax.pb.h" -#include "absl/status/status.h" -#include "absl/types/optional.h" -#include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" - -namespace google::api::expr::runtime { - -class JumpStepBase : public ExpressionStepBase { - public: - explicit JumpStepBase(absl::optional jump_offset = absl::nullopt) - : ExpressionStepBase(), jump_offset_(jump_offset) {} - - void set_jump_offset(int offset) { jump_offset_ = offset; } - - absl::Status Jump(ExecutionFrame* frame) const { - if (!jump_offset_.has_value()) { - return absl::Status(absl::StatusCode::kInternal, "Jump offset not set"); - } - return frame->JumpTo(jump_offset_.value()); - } - - private: - absl::optional jump_offset_; -}; - -// Factory method for Jump step. -std::unique_ptr CreateJumpStep( - absl::optional jump_offset = absl::nullopt); - -// Factory method for Conditional Jump step (used for and/or shortcircuiting). -// Conditional Jump requires a boolean value to sit on the stack. -// It is compared to jump_condition, and if matched, jump is performed. -// The boolean value is left on top of the stack. -std::unique_ptr CreateCondJumpStep( - bool jump_condition, absl::optional jump_offset, - size_t expected_stack_size); - -// Factory method for Ternary Conditional Jump step. -// Requires a boolean condition value on top of the stack. -// If the boolean value is false, a jump is performed to the second branch. -// The condition value is popped from the stack before jumping or continuing. -std::unique_ptr CreateTernaryCondJumpStep( - absl::optional jump_offset = absl::nullopt); - -// Factory method for ErrorJump step. -// This step performs a Jump when an Error is on the top of the stack. -// Value is left on stack if it is a bool or an error. -std::unique_ptr CreateBoolCheckJumpStep( - absl::optional jump_offset = absl::nullopt); - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_JUMP_STEP_H_ diff --git a/eval/eval/optional_or_step.cc b/eval/eval/optional_or_step.cc index e98c6ff16..551025a7e 100644 --- a/eval/eval/optional_or_step.cc +++ b/eval/eval/optional_or_step.cc @@ -21,16 +21,15 @@ #include "absl/base/optimization.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "absl/types/span.h" #include "common/casting.h" +#include "common/optional_ref.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" -#include "eval/eval/jump_step.h" +#include "eval/eval/expression_step_logic.h" #include "internal/status_macros.h" #include "runtime/internal/errors.h" #include "google/protobuf/arena.h" @@ -60,56 +59,6 @@ ErrorValue MakeNoOverloadError(OptionalOrKind kind, google::protobuf::Arena* are ABSL_UNREACHABLE(); } -// Implements short-circuiting for optional.or. -// Expected layout if short-circuiting enabled: -// -// +--------+-----------------------+-------------------------------+ -// | idx | Step | Stack After | -// +--------+-----------------------+-------------------------------+ -// | 1 | | OptionalValue | -// +--------+-----------------------+-------------------------------+ -// | 2 | Jump to 5 if present | OptionalValue | -// +--------+-----------------------+-------------------------------+ -// | 3 | | OptionalValue, OptionalValue | -// +--------+-----------------------+-------------------------------+ -// | 4 | optional.or | OptionalValue | -// +--------+-----------------------+-------------------------------+ -// | 5 | | ... | -// +--------------------------------+-------------------------------+ -// -// If implementing the orValue variant, the jump step handles unwrapping ( -// getting the result of optional.value()) -class OptionalHasValueJumpStep final : public JumpStepBase { - public: - explicit OptionalHasValueJumpStep(OptionalOrKind kind) - : JumpStepBase(std::nullopt), kind_(kind) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - const auto& value = frame->value_stack().Peek(); - auto optional_value = As(value); - // We jump if the receiver is `optional_type` which has a value or the - // receiver is an error/unknown. Unlike `_||_` we are not commutative. If - // we run into an error/unknown, we skip the `else` branch. - const bool should_jump = - (optional_value.has_value() && optional_value->HasValue()) || - (!optional_value.has_value() && (cel::InstanceOf(value) || - cel::InstanceOf(value))); - if (should_jump) { - if (kind_ == OptionalOrKind::kOrValue && optional_value.has_value()) { - frame->value_stack().PopAndPush(optional_value->Value()); - } - return Jump(frame); - } - return absl::OkStatus(); - } - - private: - const OptionalOrKind kind_; -}; - class OptionalOrStep : public ExpressionStepBase { public: explicit OptionalOrStep(OptionalOrKind kind) @@ -276,9 +225,35 @@ absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, } // namespace -std::unique_ptr CreateOptionalHasValueJumpStep(bool or_value) { - return std::make_unique( - or_value ? OptionalOrKind::kOrValue : OptionalOrKind::kOrOptional); +absl::Status OptionalHasValueJumpStep::Evaluate(ExecutionFrame* frame) const { + if (!frame->value_stack().HasEnough(1)) { + return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + } + const Value& value = frame->value_stack().Peek(); + cel::optional_ref optional_value = + As(value); + // We jump if the receiver is `optional_type` which has a value or the + // receiver is an error/unknown. Unlike `_||_` we are not commutative. If + // we run into an error/unknown, we skip the `else` branch. + const bool should_jump = + (optional_value.has_value() && optional_value->HasValue()) || + (!optional_value.has_value() && (cel::InstanceOf(value) || + cel::InstanceOf(value))); + if (should_jump) { + if (is_or_value_ && optional_value.has_value()) { + frame->value_stack().PopAndPush(optional_value->Value()); + } + if (!jump_offset_.has_value()) { + return absl::Status(absl::StatusCode::kInternal, "Jump offset not set"); + } + return frame->JumpTo(*jump_offset_); + } + return absl::OkStatus(); +} + +std::unique_ptr CreateOptionalHasValueJumpStep( + bool or_value) { + return std::make_unique(or_value); } std::unique_ptr CreateOptionalOrStep(bool is_or_value) { diff --git a/eval/eval/optional_or_step.h b/eval/eval/optional_or_step.h index 3f149e44b..1812c8090 100644 --- a/eval/eval/optional_or_step.h +++ b/eval/eval/optional_or_step.h @@ -17,13 +17,29 @@ #include #include +#include +#include "absl/status/status.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/jump_step.h" +#include "eval/eval/expression_step_base.h" namespace google::api::expr::runtime { +class OptionalHasValueJumpStep final : public ExpressionStepBase { + public: + explicit OptionalHasValueJumpStep(bool is_or_value) + : is_or_value_(is_or_value) {} + + void set_jump_offset(int offset) { jump_offset_ = offset; } + + absl::Status Evaluate(ExecutionFrame* frame) const override; + + private: + const bool is_or_value_; + std::optional jump_offset_; +}; + // Factory method for OptionalHasValueJump step, used to implement // short-circuiting optional.or and optional.orValue. // @@ -31,7 +47,8 @@ namespace google::api::expr::runtime { // true, performs a jump. If `or_value` is true and we are jumping, // `optional.value` is called and the result replaces the optional at the top of // the stack. -std::unique_ptr CreateOptionalHasValueJumpStep(bool or_value); +std::unique_ptr CreateOptionalHasValueJumpStep( + bool or_value); // Factory method for OptionalOr step, used to implement optional.or and // optional.orValue. From ffe1fe468b8a3cfcfe7befdc50ecddc1230b5172 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 1 Oct 2026 09:15:46 -0700 Subject: [PATCH 252/350] Add instructions for special cased and internal functions (==,!=,@in,#list_append,#mutable_list). PiperOrigin-RevId: 991719188 --- eval/compiler/flat_expr_builder.cc | 29 +++++----- eval/eval/BUILD | 22 ++------ eval/eval/create_list_step.cc | 18 ------- eval/eval/create_list_step.h | 6 --- eval/eval/equality_steps.cc | 87 +++++++++++++----------------- eval/eval/equality_steps.h | 6 +-- eval/eval/equality_steps_test.cc | 10 ++-- eval/eval/evaluator_core.cc | 59 ++++++++++++++++++++ eval/eval/evaluator_core.h | 34 ++++++++++++ eval/eval/evaluator_stack.h | 6 +-- 10 files changed, 160 insertions(+), 117 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index c84820d35..ff72f5f2b 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -1568,7 +1568,7 @@ class FlatExprVisitor : public cel::AstVisitor { SetRecursiveStep(CreateDirectMutableListStep(expr.id()), 1); return; } - AddStep(CreateMutableListStep(), expr.id()); + AddStep(ExpressionStep::MakeNewMutableListStep(expr.id())); return; } if (GetOptimizableListAppendOperand(comprehension.comprehension) == @@ -2143,21 +2143,25 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleListAppend( const cel::ComprehensionExpr* comprehension = comprehension_stack_.back().comprehension; const cel::Expr& loop_step = comprehension->loop_step(); + // Macro loop_step for a map() will contain a list concat operation: // accu_var + [elem] - if (&loop_step == &expr) { - AddResolvedFunctionStep(&call_expr, &expr, - cel::builtin::kRuntimeListAppend); - return CallHandlerResult::kIntercepted; - } + const bool is_map_loop_step = &loop_step == &expr; + // Macro loop_step for a filter() will contain a ternary: // filter ? accu_var + [elem] : accu_var - if (loop_step.has_call_expr() && + const bool is_filter_loop_step = + loop_step.has_call_expr() && loop_step.call_expr().function() == cel::builtin::kTernary && loop_step.call_expr().args().size() == 3 && - &(loop_step.call_expr().args()[1]) == &expr) { - AddResolvedFunctionStep(&call_expr, &expr, - cel::builtin::kRuntimeListAppend); + &(loop_step.call_expr().args()[1]) == &expr; + if (is_map_loop_step || is_filter_loop_step) { + if (RecursionEligible().has_value()) { + AddResolvedFunctionStep(&call_expr, &expr, + cel::builtin::kRuntimeListAppend); + } else { + AddStep(ExpressionStep::MakeMutableListAppendStep(expr.id())); + } return CallHandlerResult::kIntercepted; } } @@ -2186,7 +2190,8 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleHeterogeneousEquality( *depth + 1); return CallHandlerResult::kIntercepted; } - AddStep(CreateEqualityStep(inequality), expr.id()); + AddStep(inequality ? ExpressionStep::MakeFastNotEqualStep(expr.id()) + : ExpressionStep::MakeFastEqualStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -2211,7 +2216,7 @@ FlatExprVisitor::HandleHeterogeneousEqualityIn(const cel::Expr& expr, return CallHandlerResult::kIntercepted; } - AddStep(CreateInStep(), expr.id()); + AddStep(ExpressionStep::MakeFastInStep(expr.id())); return CallHandlerResult::kIntercepted; } diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 18374ab01..84d0d24b8 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -36,12 +36,14 @@ cc_library( name = "evaluator_core", srcs = [ "comprehension_step.cc", + "equality_steps.cc", "evaluator_core.cc", "lazy_init_step.cc", "logic_step.cc", ], hdrs = [ "comprehension_step.h", + "equality_steps.h", "evaluator_core.h", "lazy_init_step.h", "logic_step.h", @@ -62,12 +64,14 @@ cc_library( "//common:value", "//common:value_kind", "//eval/internal:errors", + "//internal:number", "//internal:status_macros", "//runtime", "//runtime:activation_interface", "//runtime:runtime_options", "//runtime/internal:activation_attribute_matcher_access", "//runtime/internal:errors", + "//runtime/standard:equality_functions", "@com_google_absl//absl/base", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", @@ -440,26 +444,8 @@ cc_library( cc_library( name = "equality_steps", - srcs = [ - "equality_steps.cc", - ], - hdrs = [ - "equality_steps.h", - ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", - ":expression_step_base", - "//base:builtins", - "//common:value", - "//common:value_kind", - "//internal:number", - "//internal:status_macros", - "//runtime/internal:errors", - "//runtime/standard:equality_functions", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", ], ) diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index 54c9ee8ba..9f5483a68 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -221,20 +221,6 @@ class CreateListDirectStep : public DirectExpressionStep { absl::flat_hash_set optional_indices_; }; -class MutableListStep : public ExpressionStepBase { - public: - MutableListStep() = default; - - absl::Status Evaluate(ExecutionFrame* frame) const override; -}; - -absl::Status MutableListStep::Evaluate(ExecutionFrame* frame) const { - frame->value_stack().Push(cel::CustomListValue( - cel::common_internal::NewMutableListValue(frame->arena()), - frame->arena())); - return absl::OkStatus(); -} - class DirectMutableListStep : public DirectExpressionStep { public: explicit DirectMutableListStep(int64_t expr_id) @@ -268,10 +254,6 @@ absl::StatusOr> CreateCreateListStep( MakeOptionalIndicesSet(create_list_expr)); } -std::unique_ptr CreateMutableListStep() { - return std::make_unique(); -} - std::unique_ptr CreateDirectMutableListStep( int64_t expr_id) { return std::make_unique(expr_id); diff --git a/eval/eval/create_list_step.h b/eval/eval/create_list_step.h index c377338c5..3423febe5 100644 --- a/eval/eval/create_list_step.h +++ b/eval/eval/create_list_step.h @@ -22,12 +22,6 @@ std::unique_ptr CreateDirectListStep( absl::StatusOr> CreateCreateListStep( const cel::ListExpr& create_list_expr); -// Factory method for CreateList which constructs a mutable list. -// -// This is intended for the list construction step is generated for a -// list-building comprehension (rather than a user authored expression). -std::unique_ptr CreateMutableListStep(); - // Factory method for CreateList which constructs a mutable list. // // This is intended for the list construction step is generated for a diff --git a/eval/eval/equality_steps.cc b/eval/eval/equality_steps.cc index 0069aef1e..483b91468 100644 --- a/eval/eval/equality_steps.cc +++ b/eval/eval/equality_steps.cc @@ -25,7 +25,6 @@ #include "eval/eval/attribute_trail.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" #include "internal/number.h" #include "internal/status_macros.h" #include "runtime/internal/errors.h" @@ -107,29 +106,6 @@ class DirectEqualityStep : public DirectExpressionStep { bool negation_; }; -class IterativeEqualityStep : public ExpressionStepBase { - public: - explicit IterativeEqualityStep(bool negation) : negation_(negation) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - auto args = frame->value_stack().GetSpan(2); - auto attrs = frame->value_stack().GetAttributeSpan(2); - - CEL_ASSIGN_OR_RETURN(Value result, - EvaluateEquality(*frame, args[0], attrs[0], args[1], - attrs[1], negation_)); - - frame->value_stack().PopAndPush(2, std::move(result)); - return absl::OkStatus(); - } - - private: - bool negation_; -}; - absl::StatusOr EvaluateInMap(ExecutionFrameBase& frame, const Value& item, const MapValue& container) { @@ -245,26 +221,45 @@ class DirectInStep : public DirectExpressionStep { std::unique_ptr container_; }; -class IterativeInStep : public ExpressionStepBase { - public: - IterativeInStep() = default; - - absl::Status Evaluate(ExecutionFrame* frame) const override { - if (!frame->value_stack().HasEnough(2)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - auto args = frame->value_stack().GetSpan(2); - auto attrs = frame->value_stack().GetAttributeSpan(2); +} // namespace - CEL_ASSIGN_OR_RETURN( - Value result, EvaluateIn(*frame, args[0], attrs[0], args[1], attrs[1])); - frame->value_stack().PopAndPush(2, std::move(result)); - return absl::OkStatus(); +void EvaluateFastEqualStep(bool negation, ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(2)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; } -}; + absl::Span args = frame.value_stack().GetSpan(2); + absl::Span attrs = + frame.value_stack().GetAttributeSpan(2); + + absl::StatusOr result = + EvaluateEquality(frame, args[0], attrs[0], args[1], attrs[1], negation); + if (!result.ok()) { + frame.Abort(std::move(result).status()); + return; + } + frame.value_stack().PopAndPush(2, *std::move(result)); +} -} // namespace +void EvaluateFastInStep(ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(2)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + absl::Span args = frame.value_stack().GetSpan(2); + absl::Span attrs = + frame.value_stack().GetAttributeSpan(2); + + absl::StatusOr result = + EvaluateIn(frame, args[0], attrs[0], args[1], attrs[1]); + if (!result.ok()) { + frame.Abort(std::move(result).status()); + return; + } + frame.value_stack().PopAndPush(2, *std::move(result)); +} // Factory method for recursive _==_ and _!=_ Execution step std::unique_ptr CreateDirectEqualityStep( @@ -274,11 +269,6 @@ std::unique_ptr CreateDirectEqualityStep( negation, expr_id); } -// Factory method for iterative _==_ and _!=_ Execution step -std::unique_ptr CreateEqualityStep(bool negation) { - return std::make_unique(negation); -} - // Factory method for recursive @in Execution step std::unique_ptr CreateDirectInStep( std::unique_ptr item, @@ -287,9 +277,4 @@ std::unique_ptr CreateDirectInStep( expr_id); } -// Factory method for iterative @in Execution step -std::unique_ptr CreateInStep() { - return std::make_unique(); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/equality_steps.h b/eval/eval/equality_steps.h index 7058f9c56..ef2c4bd20 100644 --- a/eval/eval/equality_steps.h +++ b/eval/eval/equality_steps.h @@ -28,16 +28,14 @@ std::unique_ptr CreateDirectEqualityStep( std::unique_ptr lhs, std::unique_ptr rhs, bool negation, int64_t expr_id); -// Factory method for iterative _==_/_!=_ Execution step -std::unique_ptr CreateEqualityStep(bool negation); +void EvaluateFastEqualStep(bool negation, ExecutionFrame& frame); // Factory method for recursive @in Execution step std::unique_ptr CreateDirectInStep( std::unique_ptr item, std::unique_ptr container, int64_t expr_id); -// Factory method for iterative @in Execution step -std::unique_ptr CreateInStep(); +void EvaluateFastInStep(ExecutionFrame& frame); } // namespace google::api::expr::runtime diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index 3b7e93626..9e90dc931 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -153,7 +153,7 @@ TEST(IterativeTest, PartialAttrUnknown) { std::make_unique(IntValue(1), cel::Attribute("foo")))); steps.push_back(ExpressionStep::MakeGenericStep( std::make_unique(IntValue(2)))); - steps.push_back(ExpressionStep::MakeGenericStep(CreateEqualityStep(false))); + steps.push_back(ExpressionStep::MakeFastEqualStep()); activation.SetUnknownPatterns({cel::AttributePattern( "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); @@ -184,7 +184,7 @@ TEST(IterativeTest, PartialAttrUnknownDisabled) { std::make_unique(IntValue(1), cel::Attribute("foo")))); steps.push_back(ExpressionStep::MakeGenericStep( std::make_unique(IntValue(2)))); - steps.push_back(ExpressionStep::MakeGenericStep(CreateEqualityStep(false))); + steps.push_back(ExpressionStep::MakeFastEqualStep()); activation.SetUnknownPatterns({cel::AttributePattern( "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); @@ -291,8 +291,8 @@ TEST_P(EqualsTest, Iterative) { std::make_unique(MakeValue(test_case.lhs, &arena)))); steps.push_back(ExpressionStep::MakeGenericStep( std::make_unique(MakeValue(test_case.rhs, &arena)))); - steps.push_back( - ExpressionStep::MakeGenericStep(CreateEqualityStep(test_case.negation))); + steps.push_back(test_case.negation ? ExpressionStep::MakeFastNotEqualStep() + : ExpressionStep::MakeFastEqualStep()); ExecutionFrame frame(steps, activation, opts, state); @@ -473,7 +473,7 @@ TEST_P(InTest, Iterative) { std::make_unique(MakeValue(test_case.lhs, &arena)))); steps.push_back(ExpressionStep::MakeGenericStep( std::make_unique(MakeValue(test_case.rhs, &arena)))); - steps.push_back(ExpressionStep::MakeGenericStep(CreateInStep())); + steps.push_back(ExpressionStep::MakeFastInStep()); ExecutionFrame frame(steps, activation, opts, state); diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 20357acec..a736d5ddf 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -30,9 +30,11 @@ #include "absl/strings/str_cat.h" #include "common/value.h" #include "common/value_kind.h" +#include "common/values/list_value_builder.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/comprehension_slots.h" #include "eval/eval/comprehension_step.h" +#include "eval/eval/equality_steps.h" #include "eval/eval/lazy_init_step.h" #include "eval/eval/logic_step.h" #include "internal/status_macros.h" @@ -170,6 +172,46 @@ void EvaluateTernaryJumpStep(const TernaryJumpStepInfo& step, } } +void EvaluateMutableListAppendStep(ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(2)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + absl::Span args = frame.value_stack().GetSpan(2); + if (args[0].IsError()) { + frame.value_stack().Pop(1); + return; + } + if (args[1].IsError()) { + frame.value_stack().SwapAndPop(2, 1); + return; + } + if (frame.unknown_processing_enabled()) { + absl::optional unknown_set = + frame.attribute_utility().IdentifyAndMergeUnknowns( + args, frame.value_stack().GetAttributeSpan(2), + /*use_partial=*/true); + if (unknown_set.has_value()) { + frame.value_stack().PopAndPush(2, std::move(*unknown_set)); + return; + } + } + if (const cel::common_internal::MutableListValue* mutable_list_value = + cel::common_internal::AsMutableListValue(args[0]); + mutable_list_value != nullptr) { + absl::Status status = mutable_list_value->Append(args[1]); + if (!status.ok()) { + frame.Abort(std::move(status)); + return; + } + frame.value_stack().Pop(1); + return; + } + frame.Abort( + absl::InvalidArgumentError("Unexpected call to runtime list append.")); +} + } // namespace void ExpressionStep::Evaluate(ExecutionFrame* context) const { @@ -252,6 +294,23 @@ void ExpressionStep::Evaluate(ExecutionFrame* context) const { << "FixedJumpStep did not have a value set."; context->JumpToOrAbort(u_.fixed_jump_step.offset); break; + case ExpressionStepKind::kFastIn: + EvaluateFastInStep(*context); + break; + case ExpressionStepKind::kFastEqual: + EvaluateFastEqualStep(/*negation=*/false, *context); + break; + case ExpressionStepKind::kFastNotEqual: + EvaluateFastEqualStep(/*negation=*/true, *context); + break; + case ExpressionStepKind::kNewMutableList: + context->value_stack().Push(cel::CustomListValue( + cel::common_internal::NewMutableListValue(context->arena()), + context->arena())); + break; + case ExpressionStepKind::kMutableListAppend: + EvaluateMutableListAppendStep(*context); + break; case ExpressionStepKind::kMovedFrom: context->Abort( absl::InternalError("ExpressionStep::Evaluate called on moved-from " diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 31ef7fdfd..4c0e2a5c4 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -87,10 +87,19 @@ enum class ExpressionStepKind : uint16_t { kComprehensionNext2 = 18, kComprehensionCond2 = 19, kReadSlot = 20, + // Jumps steps. kBooleanOrJump = 21, kBooleanAndJump = 22, kTernaryJump = 23, kFixedJump = 24, + // fast built-ins. These are used if we know they haven't been extended. + // otherwise we use normal function call steps. + kFastIn = 25, + kFastEqual = 26, + kFastNotEqual = 27, + // Special built-in steps for mutable lists implementing map/filter. + kNewMutableList = 28, + kMutableListAppend = 29, }; struct BoolJumpStepInfo { @@ -267,6 +276,26 @@ class ExpressionStep { return step; } + static ExpressionStep MakeFastInStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kFastIn, id); + } + + static ExpressionStep MakeFastEqualStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kFastEqual, id); + } + + static ExpressionStep MakeFastNotEqualStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kFastNotEqual, id); + } + + static ExpressionStep MakeNewMutableListStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kNewMutableList, id); + } + + static ExpressionStep MakeMutableListAppendStep(int64_t id = -1) { + return ExpressionStep(ExpressionStepKind::kMutableListAppend, id); + } + private: struct Header { ExpressionStepKind kind; @@ -902,6 +931,11 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kBooleanAndJump: case ExpressionStepKind::kTernaryJump: case ExpressionStepKind::kFixedJump: + case ExpressionStepKind::kFastIn: + case ExpressionStepKind::kFastEqual: + case ExpressionStepKind::kFastNotEqual: + case ExpressionStepKind::kNewMutableList: + case ExpressionStepKind::kMutableListAppend: break; default: ABSL_UNREACHABLE(); diff --git a/eval/eval/evaluator_stack.h b/eval/eval/evaluator_stack.h index b6abd1f76..e318f00ac 100644 --- a/eval/eval/evaluator_stack.h +++ b/eval/eval/evaluator_stack.h @@ -278,9 +278,9 @@ class EvaluatorStack { PopAndPush(n, std::forward(value), absl::nullopt); } - // Swaps the `n - i` element (from the top of the stack) with the `n` element, - // and pops `n - 1` elements. This results in the `n - i` element being at the - // top of the stack. + // Given the top `n` the elements of the stack, swap the `i`th element with + // the 0th, then pop n - 1 elements (leaving the `i`th element as the new + // top). void SwapAndPop(size_t n, size_t i) { ABSL_DCHECK_GT(n, 0); ABSL_DCHECK_LT(i, n); From 23771a104f167f8b2eeb38e03d8109057ef2ab98 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 1 Oct 2026 13:05:29 -0700 Subject: [PATCH 253/350] Rework function step implementation and add instructions for lazy and eager resolved functions. PiperOrigin-RevId: 991855756 --- eval/compiler/flat_expr_builder.cc | 12 +- eval/eval/BUILD | 39 ++---- eval/eval/evaluator_core.cc | 6 + eval/eval/evaluator_core.h | 43 ++++-- eval/eval/function_step.cc | 206 +++++++++++------------------ eval/eval/function_step.h | 82 +++++++++++- eval/eval/function_step_test.cc | 21 ++- 7 files changed, 224 insertions(+), 185 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index ff72f5f2b..7b9f6dabf 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -1700,9 +1700,10 @@ class FlatExprVisitor : public cel::AstVisitor { *depth + 1); return; } - AddStep( - CreateFunctionStep(*call_expr, expr->id(), std::move(lazy_overloads)), - expr->id()); + AddStep(ExpressionStep::MakeLazyFunctionStep( + CreateLazyFunctionStep(*call_expr, expr->id(), + std::move(lazy_overloads)), + expr->id())); return; } @@ -1736,8 +1737,9 @@ class FlatExprVisitor : public cel::AstVisitor { *recursion_depth + 1); return; } - AddStep(CreateFunctionStep(*call_expr, expr->id(), std::move(overloads)), - expr->id()); + AddStep(ExpressionStep::MakeEagerFunctionStep( + CreateFunctionStep(*call_expr, expr->id(), std::move(overloads)), + expr->id())); } // Add a step to the program, taking ownership. If successful, returns the diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 84d0d24b8..26b581ef4 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -38,6 +38,7 @@ cc_library( "comprehension_step.cc", "equality_steps.cc", "evaluator_core.cc", + "function_step.cc", "lazy_init_step.cc", "logic_step.cc", ], @@ -45,6 +46,7 @@ cc_library( "comprehension_step.h", "equality_steps.h", "evaluator_core.h", + "function_step.h", "lazy_init_step.h", "logic_step.h", ], @@ -60,6 +62,9 @@ cc_library( "//base:builtins", "//base:data", "//common:casting", + "//common:expr", + "//common:function_descriptor", + "//common:kind", "//common:native_type", "//common:value", "//common:value_kind", @@ -68,6 +73,10 @@ cc_library( "//internal:status_macros", "//runtime", "//runtime:activation_interface", + "//runtime:function", + "//runtime:function_overload_reference", + "//runtime:function_provider", + "//runtime:function_registry", "//runtime:runtime_options", "//runtime/internal:activation_attribute_matcher_access", "//runtime/internal:errors", @@ -75,6 +84,7 @@ cc_library( "@com_google_absl//absl/base", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:inlined_vector", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", @@ -303,37 +313,8 @@ cc_library( cc_library( name = "function_step", - srcs = [ - "function_step.cc", - ], - hdrs = [ - "function_step.h", - ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", - ":expression_step_base", - "//common:casting", - "//common:expr", - "//common:function_descriptor", - "//common:kind", - "//common:value", - "//common:value_kind", - "//eval/internal:errors", - "//internal:status_macros", - "//runtime:activation_interface", - "//runtime:function", - "//runtime:function_overload_reference", - "//runtime:function_provider", - "//runtime:function_registry", - "//runtime/internal:errors", - "@com_google_absl//absl/container:inlined_vector", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", ], ) diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index a736d5ddf..d4be67ba8 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -294,6 +294,12 @@ void ExpressionStep::Evaluate(ExecutionFrame* context) const { << "FixedJumpStep did not have a value set."; context->JumpToOrAbort(u_.fixed_jump_step.offset); break; + case ExpressionStepKind::kEagerFunction: + u_.eager_function_step->Evaluate(*context); + break; + case ExpressionStepKind::kLazyFunction: + u_.lazy_function_step->Evaluate(*context); + break; case ExpressionStepKind::kFastIn: EvaluateFastInStep(*context); break; diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 4c0e2a5c4..79b573b8e 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -39,6 +39,7 @@ #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_stack.h" #include "eval/eval/expression_step_logic.h" +#include "eval/eval/function_step.h" #include "eval/eval/iterator_stack.h" #include "eval/eval/lazy_init_step.h" #include "eval/eval/logic_step.h" @@ -71,35 +72,39 @@ enum class ExpressionStepKind : uint16_t { kUintConstant = 6, // Any constant that can't be inlined. kOtherConstant = 7, + // Lazy subexpressions. kLazyInit = 8, kAssignSlotAndPop = 9, kClearSlots = 10, + // Core boolean logic. kBooleanNot = 11, kNotStrictlyFalse = 12, kBooleanOr = 13, kBooleanAnd = 14, // Comprehension steps. - // Init step doesn't fit inline and is slow anyway, so just use a generic - // step. + // Init step doesn't fit inline and is slow anyway, so it uses a generic step. kComprehensionFinish = 15, kComprehensionNext = 16, kComprehensionCond = 17, kComprehensionNext2 = 18, kComprehensionCond2 = 19, kReadSlot = 20, - // Jumps steps. + // Jump steps. kBooleanOrJump = 21, kBooleanAndJump = 22, kTernaryJump = 23, kFixedJump = 24, + // Functions calls. + kEagerFunction = 25, + kLazyFunction = 26, // fast built-ins. These are used if we know they haven't been extended. // otherwise we use normal function call steps. - kFastIn = 25, - kFastEqual = 26, - kFastNotEqual = 27, + kFastIn = 27, + kFastEqual = 28, + kFastNotEqual = 29, // Special built-in steps for mutable lists implementing map/filter. - kNewMutableList = 28, - kMutableListAppend = 29, + kNewMutableList = 30, + kMutableListAppend = 31, }; struct BoolJumpStepInfo { @@ -296,6 +301,20 @@ class ExpressionStep { return ExpressionStep(ExpressionStepKind::kMutableListAppend, id); } + static ExpressionStep MakeEagerFunctionStep( + std::unique_ptr step_impl, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kEagerFunction, id); + step.u_.eager_function_step = step_impl.release(); + return step; + } + + static ExpressionStep MakeLazyFunctionStep( + std::unique_ptr step_impl, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kLazyFunction, id); + step.u_.lazy_function_step = step_impl.release(); + return step; + } + private: struct Header { ExpressionStepKind kind; @@ -355,6 +374,8 @@ class ExpressionStep { BoolJumpStepInfo bool_jump_step; TernaryJumpStepInfo ternary_jump_step; FixedJumpStepInfo fixed_jump_step; + EagerFunctionStep* eager_function_step; + LazyFunctionStep* lazy_function_step; Data() : empty(nullptr) {} ~Data() {} @@ -908,6 +929,12 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kOtherConstant: delete u_.other_val; break; + case ExpressionStepKind::kEagerFunction: + delete u_.eager_function_step; + break; + case ExpressionStepKind::kLazyFunction: + delete u_.lazy_function_step; + break; case ExpressionStepKind::kMovedFrom: case ExpressionStepKind::kIntConstant: case ExpressionStepKind::kBoolConstant: diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index ca63392c7..fe06b8eca 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +25,7 @@ #include "eval/eval/attribute_trail.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" #include "internal/status_macros.h" #include "runtime/activation_interface.h" @@ -146,40 +147,6 @@ bool IsUnknownFunctionResultError(const Value& result) { // resolve to a single function implementation and a descriptor or none. using ResolveResult = absl::optional; -// Implementation of ExpressionStep that finds suitable CelFunction overload and -// invokes it. Abstract base class standardizes behavior between lazy and eager -// function bindings. Derived classes provide ResolveFunction behavior. -class AbstractFunctionStep : public ExpressionStepBase { - public: - // Constructs FunctionStep that uses overloads specified. - AbstractFunctionStep(const std::string& name, size_t num_arguments, - bool receiver_style, int64_t expr_id) - : ExpressionStepBase(expr_id), - name_(name), - num_arguments_(num_arguments), - receiver_style_(receiver_style), - expr_id_(expr_id) {} - - absl::Status Evaluate(ExecutionFrame* frame) const override; - - // Handles overload resolution and updating result appropriately. - // Shouldn't update frame state. - // - // A non-ok result is an unrecoverable error, either from an illegal - // evaluation state or forwarded from an extension function. Errors where - // evaluation can reasonably condition are returned in the result as a - // cel::ErrorValue. - absl::StatusOr DoEvaluate(ExecutionFrame* frame) const; - - virtual absl::StatusOr ResolveFunction( - absl::Span args, const ExecutionFrame* frame) const = 0; - - protected: - std::string name_; - size_t num_arguments_; - bool receiver_style_; - int64_t expr_id_; -}; inline absl::StatusOr Invoke( const cel::FunctionOverloadReference& overload, int64_t expr_id, @@ -248,49 +215,7 @@ Value NoOverloadResult(absl::string_view name, frame.arena()); } -absl::StatusOr AbstractFunctionStep::DoEvaluate( - ExecutionFrame* frame) const { - // Create Span object that contains input arguments to the function. - auto input_args = frame->value_stack().GetSpan(num_arguments_); - - std::vector unknowns_args; - // Preprocess args. If an argument is partially unknown, convert it to an - // unknown attribute set. - if (frame->enable_unknowns()) { - auto input_attrs = frame->value_stack().GetAttributeSpan(num_arguments_); - unknowns_args = CheckForPartialUnknowns(frame, input_args, input_attrs); - input_args = absl::MakeConstSpan(unknowns_args); - } - - // Derived class resolves to a single function overload or none. - CEL_ASSIGN_OR_RETURN(ResolveResult matched_function, - ResolveFunction(input_args, frame)); - - // Overload found and is allowed to consume the arguments. - if (matched_function.has_value() && - ShouldAcceptOverload(matched_function->descriptor, input_args)) { - return Invoke(*matched_function, expr_id_, input_args, *frame); - } - - return NoOverloadResult(name_, input_args, receiver_style_, *frame); -} - -absl::Status AbstractFunctionStep::Evaluate(ExecutionFrame* frame) const { - if (!frame->value_stack().HasEnough(num_arguments_)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); - } - - // DoEvaluate may return a status for non-recoverable errors (e.g. - // unexpected typing, illegal expression state). Application errors that can - // reasonably be handled as a cel error will appear in the result value. - CEL_ASSIGN_OR_RETURN(auto result, DoEvaluate(frame)); - - frame->value_stack().PopAndPush(num_arguments_, std::move(result)); - - return absl::OkStatus(); -} - -absl::StatusOr ResolveStatic( +ResolveResult ResolveStatic( absl::Span input_args, absl::Span overloads) { for (const auto& overload : overloads) { @@ -341,48 +266,7 @@ absl::StatusOr ResolveLazy( return result; } -class EagerFunctionStep : public AbstractFunctionStep { - public: - EagerFunctionStep(std::vector overloads, - const std::string& name, size_t num_args, - bool receiver_style, int64_t expr_id) - : AbstractFunctionStep(name, num_args, receiver_style, expr_id), - overloads_(std::move(overloads)) {} - - absl::StatusOr ResolveFunction( - absl::Span input_args, - const ExecutionFrame* frame) const override { - return ResolveStatic(input_args, overloads_); - } - - private: - std::vector overloads_; -}; - -class LazyFunctionStep : public AbstractFunctionStep { - public: - // Constructs LazyFunctionStep that attempts to lookup function implementation - // at runtime. - LazyFunctionStep(const std::string& name, size_t num_args, - bool receiver_style, - std::vector providers, - int64_t expr_id) - : AbstractFunctionStep(name, num_args, receiver_style, expr_id), - providers_(std::move(providers)) {} - - absl::StatusOr ResolveFunction( - absl::Span input_args, - const ExecutionFrame* frame) const override; - - private: - std::vector providers_; -}; -absl::StatusOr LazyFunctionStep::ResolveFunction( - absl::Span input_args, - const ExecutionFrame* frame) const { - return ResolveLazy(input_args, name_, receiver_style_, providers_, *frame); -} class StaticResolver { public: @@ -495,6 +379,74 @@ class DirectFunctionStepImpl : public DirectExpressionStep { } // namespace +template +void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame) { + if (!frame.value_stack().HasEnough(step->num_arguments_)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + + // Create Span object that contains input arguments to the function. + absl::Span input_args = + frame.value_stack().GetSpan(step->num_arguments_); + + std::vector unknowns_args; + // Preprocess args. If an argument is partially unknown, convert it to an + // unknown attribute set. + if (frame.enable_unknowns()) { + absl::Span input_attrs = + frame.value_stack().GetAttributeSpan(step->num_arguments_); + unknowns_args = CheckForPartialUnknowns(&frame, input_args, input_attrs); + input_args = absl::MakeConstSpan(unknowns_args); + } + + // Derived class resolves to a single function overload or none. + absl::StatusOr matched_function = + step->ResolveFunction(input_args, frame); + if (!matched_function.ok()) { + frame.Abort(std::move(matched_function).status()); + return; + } + + // Overload found and is allowed to consume the arguments. + if (matched_function->has_value() && + ShouldAcceptOverload((*matched_function)->descriptor, input_args)) { + absl::StatusOr result = + Invoke(**matched_function, step->expr_id_, input_args, frame); + if (!result.ok()) { + frame.Abort(std::move(result).status()); + return; + } + frame.value_stack().PopAndPush(step->num_arguments_, *std::move(result)); + return; + } + + frame.value_stack().PopAndPush( + step->num_arguments_, + NoOverloadResult(step->name_, input_args, step->receiver_style_, frame)); +} + +void EagerFunctionStep::Evaluate(ExecutionFrame& frame) const { + EvaluateFunctionStep(this, frame); +} + +void LazyFunctionStep::Evaluate(ExecutionFrame& frame) const { + EvaluateFunctionStep(this, frame); +} + +ResolveResult EagerFunctionStep::ResolveFunction( + absl::Span input_args, + const ExecutionFrameBase& frame) const { + return ResolveStatic(input_args, overloads_); +} + +absl::StatusOr LazyFunctionStep::ResolveFunction( + absl::Span input_args, + const ExecutionFrameBase& frame) const { + return ResolveLazy(input_args, name_, receiver_style_, providers_, frame); +} + std::unique_ptr CreateDirectFunctionStep( int64_t expr_id, const cel::CallExpr& call, std::vector> deps, @@ -513,24 +465,24 @@ std::unique_ptr CreateDirectLazyFunctionStep( LazyResolver(std::move(providers), call.function(), call.has_target())); } -absl::StatusOr> CreateFunctionStep( +std::unique_ptr CreateLazyFunctionStep( const cel::CallExpr& call_expr, int64_t expr_id, std::vector lazy_overloads) { bool receiver_style = call_expr.has_target(); size_t num_args = call_expr.args().size() + (receiver_style ? 1 : 0); - const std::string& name = call_expr.function(); - return std::make_unique(name, num_args, receiver_style, - std::move(lazy_overloads), expr_id); + return std::make_unique(std::move(lazy_overloads), + call_expr.function(), num_args, + receiver_style, expr_id); } -absl::StatusOr> CreateFunctionStep( +std::unique_ptr CreateFunctionStep( const cel::CallExpr& call_expr, int64_t expr_id, std::vector overloads) { bool receiver_style = call_expr.has_target(); size_t num_args = call_expr.args().size() + (receiver_style ? 1 : 0); - const std::string& name = call_expr.function(); - return std::make_unique(std::move(overloads), name, - num_args, receiver_style, expr_id); + return std::make_unique(std::move(overloads), + call_expr.function(), num_args, + receiver_style, expr_id); } } // namespace google::api::expr::runtime diff --git a/eval/eval/function_step.h b/eval/eval/function_step.h index 94307609e..e0373d670 100644 --- a/eval/eval/function_step.h +++ b/eval/eval/function_step.h @@ -1,14 +1,21 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_FUNCTION_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_FUNCTION_STEP_H_ +#include #include #include +#include +#include +#include #include +#include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" #include "common/expr.h" +#include "common/value.h" #include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "runtime/function_overload_reference.h" #include "runtime/function_registry.h" @@ -30,19 +37,88 @@ std::unique_ptr CreateDirectLazyFunctionStep( std::vector> deps, std::vector providers); +class LazyFunctionStep; +class EagerFunctionStep; +class ExecutionFrameBase; +class ExecutionFrame; + // Factory method for Call-based execution step where the function will be // resolved at runtime (lazily) from an input Activation. -absl::StatusOr> CreateFunctionStep( +std::unique_ptr CreateLazyFunctionStep( const cel::CallExpr& call, int64_t expr_id, std::vector lazy_overloads); // Factory method for Call-based execution step where the function has been // statically resolved from a set of eagerly functions configured in the // CelFunctionRegistry. -absl::StatusOr> CreateFunctionStep( +std::unique_ptr CreateFunctionStep( const cel::CallExpr& call, int64_t expr_id, std::vector overloads); +// Common base class for EagerFunctionStep and LazyFunctionStep. +class FunctionStepBase { + private: + friend class EagerFunctionStep; + friend class LazyFunctionStep; + template + friend void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame); + + // Constructs FunctionStep that uses overloads specified. + FunctionStepBase(const std::string& name, size_t num_arguments, + bool receiver_style, int64_t expr_id) + : name_(name), + num_arguments_(num_arguments), + receiver_style_(receiver_style), + expr_id_(expr_id) {} + + std::string name_; + size_t num_arguments_; + bool receiver_style_; + int64_t expr_id_; +}; + +class EagerFunctionStep : public FunctionStepBase { + public: + EagerFunctionStep(std::vector overloads, + const std::string& name, size_t num_args, + bool receiver_style, int64_t expr_id) + : FunctionStepBase(name, num_args, receiver_style, expr_id), + overloads_(std::move(overloads)) {} + + void Evaluate(ExecutionFrame& frame) const; + + private: + template + friend void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame); + + std::optional ResolveFunction( + absl::Span input_args, + const ExecutionFrameBase& frame) const; + + std::vector overloads_; +}; + +class LazyFunctionStep : public FunctionStepBase { + public: + LazyFunctionStep(std::vector providers, + const std::string& name, size_t num_args, + bool receiver_style, int64_t expr_id) + : FunctionStepBase(name, num_args, receiver_style, expr_id), + providers_(std::move(providers)) {} + + void Evaluate(ExecutionFrame& frame) const; + + private: + template + friend void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame); + + absl::StatusOr> ResolveFunction( + absl::Span input_args, + const ExecutionFrameBase& frame) const; + + std::vector providers_; +}; + } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_FUNCTION_STEP_H_ diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index a436a3281..0480c1446 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -234,14 +234,13 @@ absl::StatusOr MakeTestFunctionStep( call.function(), call.has_target(), argument_matcher); int id = GetExprId(); if (!lazy_overloads.empty()) { - ABSL_ASSIGN_OR_RETURN(auto logic, - CreateFunctionStep(call, id, lazy_overloads)); - return ExpressionStep::MakeGenericStep(std::move(logic), id); + return ExpressionStep::MakeLazyFunctionStep( + CreateLazyFunctionStep(call, id, std::move(lazy_overloads)), id); } auto overloads = registry.FindStaticOverloads( call.function(), call.has_target(), argument_matcher); - ABSL_ASSIGN_OR_RETURN(auto logic, CreateFunctionStep(call, id, overloads)); - return ExpressionStep::MakeGenericStep(std::move(logic), id); + return ExpressionStep::MakeEagerFunctionStep( + CreateFunctionStep(call, id, std::move(overloads)), id); } // Test common functions with varying levels of unknown support. @@ -402,17 +401,13 @@ TEST_P(FunctionStepTest, TestNoMatchingOverloadsUnexpectedArgCount) { ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); ASSERT_OK_AND_ASSIGN(auto step2, MakeTestFunctionStep(call1, registry)); - ASSERT_OK_AND_ASSIGN( - auto step3_logic, - CreateFunctionStep(add_call, -1, - registry.FindStaticOverloads( - add_call.function(), false, - {cel::Kind::kInt64, cel::Kind::kInt64}))); - path.push_back(std::move(step0)); path.push_back(std::move(step1)); path.push_back(std::move(step2)); - path.push_back(ExpressionStep::MakeGenericStep(std::move(step3_logic))); + path.push_back(ExpressionStep::MakeEagerFunctionStep(CreateFunctionStep( + add_call, -1, + registry.FindStaticOverloads(add_call.function(), false, + {cel::Kind::kInt64, cel::Kind::kInt64})))); std::unique_ptr impl = GetExpression(std::move(path)); From 6d78f518637c1fa34e4cbf214c5a164dacebfb75 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 1 Oct 2026 13:11:08 -0700 Subject: [PATCH 254/350] Refactor remaining generic steps to report terminal errors via Frame::Abort PiperOrigin-RevId: 991859018 --- .../flat_expr_builder_extensions_test.cc | 4 +- eval/compiler/instrumentation.cc | 13 +- eval/eval/comprehension_step.cc | 52 ++++---- eval/eval/comprehension_step.h | 2 +- eval/eval/comprehension_step_test.cc | 3 +- eval/eval/container_access_step.cc | 12 +- eval/eval/create_list_step.cc | 18 +-- eval/eval/create_map_step.cc | 21 +-- eval/eval/create_struct_step.cc | 18 ++- eval/eval/equality_steps_test.cc | 3 +- eval/eval/evaluator_core.cc | 20 ++- eval/eval/evaluator_core.h | 2 +- eval/eval/evaluator_core_test.cc | 6 +- eval/eval/expression_step_logic.h | 5 +- eval/eval/ident_step.cc | 12 +- eval/eval/optional_or_step.cc | 29 +++-- eval/eval/optional_or_step.h | 2 +- eval/eval/regex_match_step.cc | 15 +-- eval/eval/select_step.cc | 122 ++++++++++-------- eval/eval/shadowable_value_step.cc | 18 +-- eval/eval/ternary_step.cc | 13 +- extensions/select_optimization.cc | 35 +++-- 22 files changed, 231 insertions(+), 194 deletions(-) diff --git a/eval/compiler/flat_expr_builder_extensions_test.cc b/eval/compiler/flat_expr_builder_extensions_test.cc index db13f6ec6..2eae196af 100644 --- a/eval/compiler/flat_expr_builder_extensions_test.cc +++ b/eval/compiler/flat_expr_builder_extensions_test.cc @@ -57,9 +57,7 @@ using ::testing::Optional; class TestStepLogic : public ExpressionStepLogic { public: - absl::Status Evaluate(ExecutionFrame* frame) const override { - return absl::OkStatus(); - } + void Evaluate(ExecutionFrame* frame) const override {} }; std::unique_ptr MakeTestStepLogic() { diff --git a/eval/compiler/instrumentation.cc b/eval/compiler/instrumentation.cc index ea3e7ddbb..7f522c739 100644 --- a/eval/compiler/instrumentation.cc +++ b/eval/compiler/instrumentation.cc @@ -37,14 +37,17 @@ class InstrumentStep : public ExpressionStepBase { expr_id_(expr_id), instrumentation_(std::move(instrumentation)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError("stack underflow in instrument step."); + frame->Abort(absl::InternalError("stack underflow in instrument step.")); + return; } - return instrumentation_(expr_id_, frame->value_stack().Peek()); - - return absl::OkStatus(); + if (absl::Status status = + instrumentation_(expr_id_, frame->value_stack().Peek()); + !status.ok()) { + frame->Abort(std::move(status)); + } } private: diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc index d67a7e8a6..6293f97f4 100644 --- a/eval/eval/comprehension_step.cc +++ b/eval/eval/comprehension_step.cc @@ -428,14 +428,16 @@ absl::Status ComprehensionDirectStep::Evaluate2(ExecutionFrameBase& frame, } // namespace -absl::Status ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { +void ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + frame->Abort(absl::InternalError("Value stack underflow")); + return; } const Value& top = frame->value_stack().Peek(); if (top.IsError() || top.IsUnknown()) { - return frame->JumpTo(error_jump_offset_); + frame->JumpToOrAbort(error_jump_offset_); + return; } if (frame->enable_unknowns() && top.IsMap()) { @@ -443,40 +445,38 @@ absl::Status ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { if (frame->attribute_utility().CheckForUnknownPartial(top_attr)) { frame->value_stack().PopAndPush( frame->attribute_utility().CreateUnknownSet(top_attr.attribute())); - return frame->JumpTo(error_jump_offset_); + frame->JumpToOrAbort(error_jump_offset_); + return; } } + absl::StatusOr iterator; switch (top.kind()) { - case ValueKind::kList: { - CEL_ASSIGN_OR_RETURN(auto iterator, top.GetList().NewIterator()); - if (has_iter2_) { - frame->iterator_stack().Push(std::move(iterator), iter_slot_, - iter2_slot_, accu_slot_); - } else { - frame->iterator_stack().Push(std::move(iterator), iter_slot_, - accu_slot_); - } - } break; - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN(auto iterator, top.GetMap().NewIterator()); - if (has_iter2_) { - frame->iterator_stack().Push(std::move(iterator), iter_slot_, - iter2_slot_, accu_slot_); - } else { - frame->iterator_stack().Push(std::move(iterator), iter_slot_, - accu_slot_); - } - } break; + case ValueKind::kList: + iterator = top.GetList().NewIterator(); + break; + case ValueKind::kMap: + iterator = top.GetMap().NewIterator(); + break; default: // Replace with an error and jump past // ComprehensionFinishStep. frame->value_stack().PopAndPush(cel::ErrorValue::From( CreateNoMatchingOverloadError(""), frame->arena())); - return frame->JumpTo(error_jump_offset_); + frame->JumpToOrAbort(error_jump_offset_); + return; } - return absl::OkStatus(); + if (!iterator.ok()) { + frame->Abort(std::move(iterator).status()); + return; + } + if (has_iter2_) { + frame->iterator_stack().Push(*std::move(iterator), iter_slot_, iter2_slot_, + accu_slot_); + } else { + frame->iterator_stack().Push(*std::move(iterator), iter_slot_, accu_slot_); + } } void ComprehensionNextStep::Evaluate1(ExecutionFrame* frame) const { diff --git a/eval/eval/comprehension_step.h b/eval/eval/comprehension_step.h index 8ec7960db..8bac8dab3 100644 --- a/eval/eval/comprehension_step.h +++ b/eval/eval/comprehension_step.h @@ -40,7 +40,7 @@ class ComprehensionInitStep final : public ExpressionStepLogic { void set_error_jump_offset(int offset) { error_jump_offset_ = offset; } - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: const size_t iter_slot_; diff --git a/eval/eval/comprehension_step_test.cc b/eval/eval/comprehension_step_test.cc index ee8b764c1..2ffbafb1b 100644 --- a/eval/eval/comprehension_step_test.cc +++ b/eval/eval/comprehension_step_test.cc @@ -81,9 +81,8 @@ class GetListKeysResultStep : public ExpressionStepBase { public: GetListKeysResultStep() : ExpressionStepBase() {} - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Pop(1); - return absl::OkStatus(); } }; diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index 8ce87ef3f..f4a33b217 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -289,17 +289,17 @@ class ContainerAccessStep : public ExpressionStepBase { explicit ContainerAccessStep(bool enable_optional_types) : ExpressionStepBase(), enable_optional_types_(enable_optional_types) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: bool enable_optional_types_; }; -absl::Status ContainerAccessStep::Evaluate(ExecutionFrame* frame) const { +void ContainerAccessStep::Evaluate(ExecutionFrame* frame) const { if (!frame->value_stack().HasEnough(kNumContainerAccessArguments)) { - return absl::Status( - absl::StatusCode::kInternal, - "Insufficient arguments supplied for ContainerAccess-type expression"); + frame->Abort(absl::InternalError( + "Insufficient arguments supplied for ContainerAccess-type expression")); + return; } Value result; @@ -312,8 +312,6 @@ absl::Status ContainerAccessStep::Evaluate(ExecutionFrame* frame) const { enable_optional_types_, result, result_trail); frame->value_stack().PopAndPush(kNumContainerAccessArguments, std::move(result), std::move(result_trail)); - - return absl::OkStatus(); } class DirectContainerAccessStep : public DirectExpressionStep { diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index 9f5483a68..65636f347 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -38,7 +38,7 @@ class CreateListStep : public ExpressionStepBase { CreateListStep(int list_size, absl::flat_hash_set optional_indices) : list_size_(list_size), optional_indices_(std::move(optional_indices)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: absl::Status DoEvaluate(ExecutionFrame* frame, Value* result) const; @@ -47,22 +47,24 @@ class CreateListStep : public ExpressionStepBase { absl::flat_hash_set optional_indices_; }; -absl::Status CreateListStep::Evaluate(ExecutionFrame* frame) const { +void CreateListStep::Evaluate(ExecutionFrame* frame) const { if (list_size_ < 0) { - return absl::Status(absl::StatusCode::kInternal, - "CreateListStep: list size is <0"); + frame->Abort(absl::InternalError("CreateListStep: list size is <0")); + return; } if (!frame->value_stack().HasEnough(list_size_)) { - return absl::Status(absl::StatusCode::kInternal, - "CreateListStep: stack underflow"); + frame->Abort(absl::InternalError("CreateListStep: stack underflow")); + return; } Value result; - CEL_RETURN_IF_ERROR(DoEvaluate(frame, &result)); + if (absl::Status status = DoEvaluate(frame, &result); !status.ok()) { + frame->Abort(std::move(status)); + return; + } frame->value_stack().PopAndPush(list_size_, std::move(result)); - return absl::OkStatus(); } absl::Status CreateListStep::DoEvaluate(ExecutionFrame* frame, diff --git a/eval/eval/create_map_step.cc b/eval/eval/create_map_step.cc index e90de1fc7..4b27e5e30 100644 --- a/eval/eval/create_map_step.cc +++ b/eval/eval/create_map_step.cc @@ -57,7 +57,7 @@ class CreateStructStepForMap final : public ExpressionStepBase { : entry_count_(entry_count), optional_indices_(std::move(optional_indices)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: absl::StatusOr DoEvaluate(ExecutionFrame* frame) const; @@ -118,16 +118,20 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( return std::move(*builder).Build(); } -absl::Status CreateStructStepForMap::Evaluate(ExecutionFrame* frame) const { +void CreateStructStepForMap::Evaluate(ExecutionFrame* frame) const { if (frame->value_stack().size() < 2 * entry_count_) { - return absl::InternalError("CreateStructStepForMap: stack underflow"); + frame->Abort( + absl::InternalError("CreateStructStepForMap: stack underflow")); + return; } - CEL_ASSIGN_OR_RETURN(auto result, DoEvaluate(frame)); - - frame->value_stack().PopAndPush(2 * entry_count_, std::move(result)); + absl::StatusOr result = DoEvaluate(frame); + if (!result.ok()) { + frame->Abort(std::move(result).status()); + return; + } - return absl::OkStatus(); + frame->value_stack().PopAndPush(2 * entry_count_, *std::move(result)); } class DirectCreateMapStep : public DirectExpressionStep { @@ -238,10 +242,9 @@ class MutableMapStep final : public ExpressionStepBase { public: MutableMapStep() = default; - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Push(cel::CustomMapValue( NewMutableMapValue(frame->arena()), frame->arena())); - return absl::OkStatus(); } }; diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index a4467a736..bce2a3ea9 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -55,7 +55,7 @@ class CreateStructStepForStruct final : public ExpressionStepBase { entries_(std::move(entries)), optional_indices_(std::move(optional_indices)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: absl::StatusOr DoEvaluate(ExecutionFrame* frame) const; @@ -131,14 +131,18 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( return std::move(*builder).Build(); } -absl::Status CreateStructStepForStruct::Evaluate(ExecutionFrame* frame) const { +void CreateStructStepForStruct::Evaluate(ExecutionFrame* frame) const { if (frame->value_stack().size() < entries_.size()) { - return absl::InternalError("CreateStructStepForStruct: stack underflow"); + frame->Abort( + absl::InternalError("CreateStructStepForStruct: stack underflow")); + return; } - CEL_ASSIGN_OR_RETURN(Value result, DoEvaluate(frame)); - frame->value_stack().PopAndPush(entries_.size(), std::move(result)); - - return absl::OkStatus(); + absl::StatusOr result = DoEvaluate(frame); + if (!result.ok()) { + frame->Abort(std::move(result).status()); + return; + } + frame->value_stack().PopAndPush(entries_.size(), *std::move(result)); } class DirectCreateStructStep : public DirectExpressionStep { diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index 9e90dc931..76031e169 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -64,9 +64,8 @@ class ValueStep : public ExpressionStepLogic, public DirectExpressionStep { value_(std::move(value)), attr_() {} - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Push(value_, attr_); - return absl::OkStatus(); } absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index d4be67ba8..77f0bb30f 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -146,8 +146,7 @@ void EvaluateTernaryJumpStep(const TernaryJumpStepInfo& step, ExecutionFrame& frame) { ABSL_DCHECK(step.set) << "TernaryJumpStep did not have a value set."; if (!frame.value_stack().HasEnough(1)) { - frame.Abort( - absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + frame.Abort(absl::InternalError("TernaryJumpStep: value stack underflow")); return; } const cel::Value& condition = frame.value_stack().Peek(); @@ -216,13 +215,9 @@ void EvaluateMutableListAppendStep(ExecutionFrame& frame) { void ExpressionStep::Evaluate(ExecutionFrame* context) const { switch (header_.kind) { - case ExpressionStepKind::kGenericLogic: { - EvaluationStatus s(u_.logic->Evaluate(context)); - if (!s.ok()) { - context->Abort(std::move(s).Consume()); - } + case ExpressionStepKind::kGenericLogic: + u_.logic->Evaluate(context); break; - } case ExpressionStepKind::kIntConstant: context->value_stack().Push(cel::IntValue(u_.int_val)); break; @@ -512,12 +507,15 @@ FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep& step) { return nullptr; } -absl::Status WrappedDirectStep::Evaluate(ExecutionFrame* frame) const { +void WrappedDirectStep::Evaluate(ExecutionFrame* frame) const { cel::Value result; AttributeTrail attribute_trail; - CEL_RETURN_IF_ERROR(impl_->Evaluate(*frame, result, attribute_trail)); + if (absl::Status status = impl_->Evaluate(*frame, result, attribute_trail); + !status.ok()) { + frame->Abort(std::move(status)); + return; + } frame->value_stack().Push(std::move(result), std::move(attribute_trail)); - return absl::OkStatus(); } } // namespace google::api::expr::runtime diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 79b573b8e..8e71fcd83 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -398,7 +398,7 @@ class WrappedDirectStep : public ExpressionStepLogic { int64_t expr_id = -1) : impl_(std::move(impl)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; cel::NativeTypeId GetNativeTypeId() const override { return cel::NativeTypeId::For(); diff --git a/eval/eval/evaluator_core_test.cc b/eval/eval/evaluator_core_test.cc index dde70857e..c67dc8b58 100644 --- a/eval/eval/evaluator_core_test.cc +++ b/eval/eval/evaluator_core_test.cc @@ -46,9 +46,8 @@ class FakeConstExpressionStep : public ExpressionStepLogic { public: FakeConstExpressionStep() = default; - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { frame->value_stack().Push(CreateIntValue(0)); - return absl::OkStatus(); } }; @@ -58,13 +57,12 @@ class FakeIncrementExpressionStep : public ExpressionStepLogic { public: FakeIncrementExpressionStep() = default; - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { auto value = frame->value_stack().Peek(); frame->value_stack().Pop(1); EXPECT_TRUE(value->Is()); int64_t val = value.GetInt().NativeValue(); frame->value_stack().Push(CreateIntValue(val + 1)); - return absl::OkStatus(); } }; diff --git a/eval/eval/expression_step_logic.h b/eval/eval/expression_step_logic.h index 2d256636f..2b07af527 100644 --- a/eval/eval/expression_step_logic.h +++ b/eval/eval/expression_step_logic.h @@ -15,7 +15,6 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_LOGIC_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_EXPRESSION_STEP_LOGIC_H_ -#include "absl/status/status.h" #include "common/native_type.h" namespace google::api::expr::runtime { @@ -35,7 +34,9 @@ class ExpressionStepLogic { // interface. // ExpressionStep instances can in specific cases // modify execution order(perform jumps). - virtual absl::Status Evaluate(ExecutionFrame* context) const = 0; + // + // Unrecoverable errors are reported via ExecutionFrame::Abort. + virtual void Evaluate(ExecutionFrame* frame) const = 0; // Return the type of the underlying expression step for special handling in // the planning phase. This should only be overridden by special cases, and diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index eee511a7e..e3985d183 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -32,7 +32,7 @@ class IdentStep : public ExpressionStepBase { public: explicit IdentStep(absl::string_view name) : name_(name) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: std::string name_; @@ -74,15 +74,17 @@ absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, return absl::OkStatus(); } -absl::Status IdentStep::Evaluate(ExecutionFrame* frame) const { +void IdentStep::Evaluate(ExecutionFrame* frame) const { Value value; AttributeTrail attribute; - CEL_RETURN_IF_ERROR(LookupIdent(name_, *frame, value, attribute)); + if (absl::Status status = LookupIdent(name_, *frame, value, attribute); + !status.ok()) { + frame->Abort(std::move(status)); + return; + } frame->value_stack().Push(std::move(value), std::move(attribute)); - - return absl::OkStatus(); } absl::StatusOr LookupSlot( diff --git a/eval/eval/optional_or_step.cc b/eval/eval/optional_or_step.cc index 551025a7e..5ac1e8400 100644 --- a/eval/eval/optional_or_step.cc +++ b/eval/eval/optional_or_step.cc @@ -64,7 +64,7 @@ class OptionalOrStep : public ExpressionStepBase { explicit OptionalOrStep(OptionalOrKind kind) : ExpressionStepBase(), kind_(kind) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: const OptionalOrKind kind_; @@ -113,9 +113,10 @@ absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, return absl::OkStatus(); } -absl::Status OptionalOrStep::Evaluate(ExecutionFrame* frame) const { +void OptionalOrStep::Evaluate(ExecutionFrame* frame) const { if (!frame->value_stack().HasEnough(2)) { - return absl::InternalError("Value stack underflow"); + frame->Abort(absl::InternalError("Value stack underflow")); + return; } absl::Span args = frame->value_stack().GetSpan(2); @@ -124,12 +125,15 @@ absl::Status OptionalOrStep::Evaluate(ExecutionFrame* frame) const { Value result; AttributeTrail result_attr; - CEL_RETURN_IF_ERROR(EvalOptionalOr(kind_, args[0], args[1], args_attr[0], - args_attr[1], result, result_attr, - frame->arena())); + if (absl::Status status = + EvalOptionalOr(kind_, args[0], args[1], args_attr[0], args_attr[1], + result, result_attr, frame->arena()); + !status.ok()) { + frame->Abort(std::move(status)); + return; + } frame->value_stack().PopAndPush(2, std::move(result), std::move(result_attr)); - return absl::OkStatus(); } class ExhaustiveDirectOptionalOrStep : public DirectExpressionStep { @@ -225,9 +229,10 @@ absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, } // namespace -absl::Status OptionalHasValueJumpStep::Evaluate(ExecutionFrame* frame) const { +void OptionalHasValueJumpStep::Evaluate(ExecutionFrame* frame) const { if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + frame->Abort(absl::InternalError("Value stack underflow")); + return; } const Value& value = frame->value_stack().Peek(); cel::optional_ref optional_value = @@ -244,11 +249,11 @@ absl::Status OptionalHasValueJumpStep::Evaluate(ExecutionFrame* frame) const { frame->value_stack().PopAndPush(optional_value->Value()); } if (!jump_offset_.has_value()) { - return absl::Status(absl::StatusCode::kInternal, "Jump offset not set"); + frame->Abort(absl::InternalError("Jump offset not set")); + return; } - return frame->JumpTo(*jump_offset_); + frame->JumpToOrAbort(*jump_offset_); } - return absl::OkStatus(); } std::unique_ptr CreateOptionalHasValueJumpStep( diff --git a/eval/eval/optional_or_step.h b/eval/eval/optional_or_step.h index 1812c8090..f54057fa9 100644 --- a/eval/eval/optional_or_step.h +++ b/eval/eval/optional_or_step.h @@ -33,7 +33,7 @@ class OptionalHasValueJumpStep final : public ExpressionStepBase { void set_jump_offset(int offset) { jump_offset_ = offset; } - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: const bool is_or_value_; diff --git a/eval/eval/regex_match_step.cc b/eval/eval/regex_match_step.cc index abc301225..d13e4f651 100644 --- a/eval/eval/regex_match_step.cc +++ b/eval/eval/regex_match_step.cc @@ -62,23 +62,22 @@ class RegexMatchStep final : public ExpressionStepBase { explicit RegexMatchStep(std::shared_ptr re2) : ExpressionStepBase(), re2_(std::move(re2)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(kNumRegexMatchArguments)) { - return absl::Status(absl::StatusCode::kInternal, - "Insufficient arguments supplied for regular " - "expression match"); + frame->Abort(absl::InternalError( + "Insufficient arguments supplied for regular expression match")); + return; } auto input_args = frame->value_stack().GetSpan(kNumRegexMatchArguments); const auto& subject = input_args[kRegexMatchStepSubject]; if (!subject->Is()) { - return absl::Status(absl::StatusCode::kInternal, - "First argument for regular " - "expression match must be a string"); + frame->Abort(absl::InternalError( + "First argument for regular expression match must be a string")); + return; } bool match = subject.GetString().NativeValue(MatchesVisitor{*re2_}); frame->value_stack().Pop(kNumRegexMatchArguments); frame->value_stack().Push(cel::BoolValue(match)); - return absl::OkStatus(); } private: diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 528f12d0f..1d6c337d6 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -215,7 +215,7 @@ class SelectStep : public ExpressionStepBase { test_field_presence_(test_field_presence), enable_optional_types_(enable_optional_types) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; protected: std::string field_; @@ -224,10 +224,11 @@ class SelectStep : public ExpressionStepBase { bool enable_optional_types_; }; -absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { +void SelectStep::Evaluate(ExecutionFrame* frame) const { if (!frame->value_stack().HasEnough(1)) { - return absl::Status(absl::StatusCode::kInternal, - "No arguments supplied for Select-type expression"); + frame->Abort(absl::InternalError( + "No arguments supplied for Select-type expression")); + return; } const Value& arg = frame->value_stack().Peek(); @@ -235,7 +236,7 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { if (arg.IsUnknown() || arg.IsError()) { // Bubble up unknowns and errors. - return absl::OkStatus(); + return; } AttributeTrail result_trail; @@ -255,7 +256,7 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { frame->value_stack().PopAndPush( cel::ErrorValue::From(InvalidSelectTargetError(), frame->arena()), std::move(result_trail)); - return absl::OkStatus(); + return; } absl::optional marked_attribute_check = @@ -263,7 +264,7 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { if (marked_attribute_check.has_value()) { frame->value_stack().PopAndPush(std::move(marked_attribute_check).value(), std::move(result_trail)); - return absl::OkStatus(); + return; } Value result; @@ -273,28 +274,32 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { if (!optional_arg->HasValue()) { frame->value_stack().PopAndPush(cel::BoolValue{false}, std::move(result_trail)); - return absl::OkStatus(); + return; } optional_arg->Value(&result); target = &result; } - CEL_RETURN_IF_ERROR( - PerformHas(*target, field_, cel::StringValue::WrapUnsafe(field_), - frame->descriptor_pool(), frame->message_factory(), - frame->arena(), result)); + if (absl::Status status = + PerformHas(*target, field_, cel::StringValue::WrapUnsafe(field_), + frame->descriptor_pool(), frame->message_factory(), + frame->arena(), result); + !status.ok()) { + frame->Abort(std::move(status)); + return; + } frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); - return absl::OkStatus(); + return; } if (optional_arg) { if (!optional_arg->HasValue()) { frame->value_stack().PopAndPush(OptionalValue::None(), std::move(result_trail)); - return absl::OkStatus(); + return; } Value value; optional_arg->Value(&value); - auto status = PerformOptionalGet( + absl::Status status = PerformOptionalGet( value, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, frame->descriptor_pool(), frame->message_factory(), frame->arena(), frame->options().enable_use_new_field_select_implementation, result); @@ -302,15 +307,18 @@ absl::Status SelectStep::Evaluate(ExecutionFrame* frame) const { result = ErrorValue::From(std::move(status), frame->arena()); } frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); - return absl::OkStatus(); + return; } - CEL_RETURN_IF_ERROR(PerformGet( - arg, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, - frame->descriptor_pool(), frame->message_factory(), frame->arena(), - frame->options().enable_use_new_field_select_implementation, result)); + if (absl::Status status = PerformGet( + arg, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, + frame->descriptor_pool(), frame->message_factory(), frame->arena(), + frame->options().enable_use_new_field_select_implementation, result); + !status.ok()) { + frame->Abort(std::move(status)); + return; + } frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); - return absl::OkStatus(); } class DirectSelectStep : public DirectExpressionStep { @@ -473,10 +481,11 @@ class ProtoSelectStep : public SelectStep { ABSL_DCHECK(field_descriptor_ != nullptr); } - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError( - "No arguments supplied for Select-type expression"); + frame->Abort(absl::InternalError( + "No arguments supplied for Select-type expression")); + return; } const Value& arg = frame->value_stack().Peek(); @@ -484,7 +493,8 @@ class ProtoSelectStep : public SelectStep { unwrapped.has_value() && SupportsCachedFieldDescriptor(*unwrapped, descriptor_, field_descriptor_)) { - return EvaluateMessageFieldGet(frame, *unwrapped); + EvaluateMessageFieldGet(frame, *unwrapped); + return; } else if (const google::protobuf::Message* legacy_message = cel::interop_internal::GetLegacyMessage(arg); frame->options().enable_use_new_field_select_implementation && @@ -494,19 +504,20 @@ class ProtoSelectStep : public SelectStep { // can minimize back and forth interop conversions. if (SupportsCachedFieldDescriptor(parsed_message, descriptor_, field_descriptor_)) { - return EvaluateMessageFieldGet(frame, legacy_message); + EvaluateMessageFieldGet(frame, legacy_message); + return; } } // If we get an unexpected value type, fall back to the generic // implementation. - return SelectStep::Evaluate(frame); + SelectStep::Evaluate(frame); } private: - absl::Status EvaluateMessageFieldGet( + void EvaluateMessageFieldGet( ExecutionFrame* frame, const cel::ParsedMessageValue& parsed_message) const; - absl::Status EvaluateMessageFieldGet( + void EvaluateMessageFieldGet( ExecutionFrame* frame, const google::protobuf::Message* absl_nonnull legacy_message) const; @@ -514,28 +525,35 @@ class ProtoSelectStep : public SelectStep { const google::protobuf::FieldDescriptor* field_descriptor_; }; -absl::Status ProtoSelectStep::EvaluateMessageFieldGet( +void ProtoSelectStep::EvaluateMessageFieldGet( ExecutionFrame* frame, const cel::ParsedMessageValue& parsed_message) const { if (CheckAttributeTrail(field_, frame)) { - return absl::OkStatus(); + return; + } + if (absl::Status status = parsed_message.GetField( + field_descriptor_, unboxing_option_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), + &frame->value_stack().Peek()); + !status.ok()) { + frame->Abort(std::move(status)); } - return parsed_message.GetField( - field_descriptor_, unboxing_option_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &frame->value_stack().Peek()); } -absl::Status ProtoSelectStep::EvaluateMessageFieldGet( +void ProtoSelectStep::EvaluateMessageFieldGet( ExecutionFrame* frame, const google::protobuf::Message* absl_nonnull legacy_message) const { ABSL_DCHECK(legacy_message != nullptr); if (CheckAttributeTrail(field_, frame)) { - return absl::OkStatus(); + return; + } + if (absl::Status status = cel::interop_internal::WrapLegacyMessageField( + legacy_message, field_descriptor_, unboxing_option_, + frame->descriptor_pool(), frame->message_factory(), frame->arena(), + &frame->value_stack().Peek()); + !status.ok()) { + frame->Abort(std::move(status)); } - return cel::interop_internal::WrapLegacyMessageField( - legacy_message, field_descriptor_, unboxing_option_, - frame->descriptor_pool(), frame->message_factory(), frame->arena(), - &frame->value_stack().Peek()); } class ProtoHasStep : public SelectStep { @@ -551,10 +569,11 @@ class ProtoHasStep : public SelectStep { ABSL_DCHECK(field_descriptor_ != nullptr); } - absl::Status Evaluate(ExecutionFrame* frame) const override { + void Evaluate(ExecutionFrame* frame) const override { if (!frame->value_stack().HasEnough(1)) { - return absl::InternalError( - "No arguments supplied for Select-type expression"); + frame->Abort(absl::InternalError( + "No arguments supplied for Select-type expression")); + return; } const Value& arg = frame->value_stack().Peek(); @@ -562,7 +581,8 @@ class ProtoHasStep : public SelectStep { unwrapped.has_value() && SupportsCachedFieldDescriptor(*unwrapped, descriptor_, field_descriptor_)) { - return EvaluateHas(frame, *unwrapped); + EvaluateHas(frame, *unwrapped); + return; } else if (const google::protobuf::Message* legacy_message = cel::interop_internal::GetLegacyMessage(arg); legacy_message != nullptr) { @@ -570,31 +590,31 @@ class ProtoHasStep : public SelectStep { cel::UnsafeParsedMessageValue(legacy_message); if (SupportsCachedFieldDescriptor(parsed_message, descriptor_, field_descriptor_)) { - return EvaluateHas(frame, parsed_message); + EvaluateHas(frame, parsed_message); + return; } } // If we get an unexpected value type, fall back to the generic // implementation. - return SelectStep::Evaluate(frame); + SelectStep::Evaluate(frame); } private: - absl::Status EvaluateHas(ExecutionFrame* frame, - const cel::ParsedMessageValue& parsed_message) const; + void EvaluateHas(ExecutionFrame* frame, + const cel::ParsedMessageValue& parsed_message) const; const google::protobuf::Descriptor* descriptor_; const google::protobuf::FieldDescriptor* field_descriptor_; }; -absl::Status ProtoHasStep::EvaluateHas( +void ProtoHasStep::EvaluateHas( ExecutionFrame* frame, const cel::ParsedMessageValue& parsed_message) const { if (CheckAttributeTrail(field_, frame)) { - return absl::OkStatus(); + return; } frame->value_stack().Peek() = BoolValue{parsed_message.HasField(field_descriptor_)}; - return absl::OkStatus(); } } // namespace diff --git a/eval/eval/shadowable_value_step.cc b/eval/eval/shadowable_value_step.cc index 0b75f7cc2..b6199ed7e 100644 --- a/eval/eval/shadowable_value_step.cc +++ b/eval/eval/shadowable_value_step.cc @@ -29,25 +29,27 @@ class ShadowableValueStep : public ExpressionStepBase { identifier_(std::move(identifier)), value_(std::move(value)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: std::string identifier_; Value value_; }; -absl::Status ShadowableValueStep::Evaluate(ExecutionFrame* frame) const { +void ShadowableValueStep::Evaluate(ExecutionFrame* frame) const { cel::Value result; - CEL_ASSIGN_OR_RETURN(auto found, - frame->modern_activation().FindVariable( - identifier_, frame->descriptor_pool(), - frame->message_factory(), frame->arena(), &result)); - if (found) { + absl::StatusOr found = frame->modern_activation().FindVariable( + identifier_, frame->descriptor_pool(), frame->message_factory(), + frame->arena(), &result); + if (!found.ok()) { + frame->Abort(std::move(found).status()); + return; + } + if (*found) { frame->value_stack().Push(std::move(result)); } else { frame->value_stack().Push(value_); } - return absl::OkStatus(); } class DirectShadowableValueStep : public DirectExpressionStep { diff --git a/eval/eval/ternary_step.cc b/eval/eval/ternary_step.cc index 97b064d22..0924b915f 100644 --- a/eval/eval/ternary_step.cc +++ b/eval/eval/ternary_step.cc @@ -128,13 +128,14 @@ class TernaryStep : public ExpressionStepBase { // Constructs FunctionStep that uses overloads specified. TernaryStep() : ExpressionStepBase() {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; }; -absl::Status TernaryStep::Evaluate(ExecutionFrame* frame) const { +void TernaryStep::Evaluate(ExecutionFrame* frame) const { // Must have 3 or more values on the stack. if (!frame->value_stack().HasEnough(3)) { - return absl::Status(absl::StatusCode::kInternal, "Value stack underflow"); + frame->Abort(absl::InternalError("Value stack underflow")); + return; } // Create Span object that contains input arguments to the function. @@ -148,13 +149,13 @@ absl::Status TernaryStep::Evaluate(ExecutionFrame* frame) const { // Check if unknown? if (condition.IsUnknown()) { frame->value_stack().Pop(2); - return absl::OkStatus(); + return; } } if (condition.IsError()) { frame->value_stack().Pop(2); - return absl::OkStatus(); + return; } cel::Value result; @@ -168,8 +169,6 @@ absl::Status TernaryStep::Evaluate(ExecutionFrame* frame) const { } frame->value_stack().PopAndPush(args.size(), std::move(result)); - - return absl::OkStatus(); } } // namespace diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 60d414de3..7bca6414c 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -755,7 +755,7 @@ class StackMachineImpl : public ExpressionStepBase { StackMachineImpl(int expr_id, OptimizedSelectImpl impl) : ExpressionStepBase(expr_id), impl_(std::move(impl)) {} - absl::Status Evaluate(ExecutionFrame* frame) const override; + void Evaluate(ExecutionFrame* frame) const override; private: // Get the effective attribute for the optimized select expression. @@ -772,7 +772,7 @@ AttributeTrail StackMachineImpl::GetAttributeTrail( return impl_.GetAttributeTrail(attr); } -absl::Status StackMachineImpl::Evaluate(ExecutionFrame* frame) const { +void StackMachineImpl::Evaluate(ExecutionFrame* frame) const { // Default empty. AttributeTrail attribute_trail; // TODO(uncreated-issue/51): add support for variable qualifiers and string literal @@ -784,7 +784,7 @@ absl::Status StackMachineImpl::Evaluate(ExecutionFrame* frame) const { if (operand->Is() || operand->Is()) { // Just forward the error which is already top of stack. - return absl::OkStatus(); + return; } if (frame->enable_attribute_tracking()) { @@ -793,27 +793,34 @@ absl::Status StackMachineImpl::Evaluate(ExecutionFrame* frame) const { // select arguments. // TODO(uncreated-issue/51): add support variable qualifiers attribute_trail = GetAttributeTrail(frame); - CEL_ASSIGN_OR_RETURN(std::optional value, - CheckForMarkedAttributes(*frame, attribute_trail)); - if (value.has_value()) { + absl::StatusOr> value = + CheckForMarkedAttributes(*frame, attribute_trail); + if (!value.ok()) { + frame->Abort(std::move(value).status()); + return; + } + if (value->has_value()) { frame->value_stack().Pop(kStackInputs); - frame->value_stack().Push(std::move(value).value(), + frame->value_stack().Push(std::move(*value).value(), std::move(attribute_trail)); - return absl::OkStatus(); + return; } } if (!operand->Is()) { - return absl::InvalidArgumentError( - "Expected struct type for select optimization."); + frame->Abort(absl::InvalidArgumentError( + "Expected struct type for select optimization.")); + return; } - CEL_ASSIGN_OR_RETURN(Value result, - impl_.ApplySelect(*frame, operand.GetStruct())); + absl::StatusOr result = impl_.ApplySelect(*frame, operand.GetStruct()); + if (!result.ok()) { + frame->Abort(std::move(result).status()); + return; + } frame->value_stack().Pop(kStackInputs); - frame->value_stack().Push(std::move(result), std::move(attribute_trail)); - return absl::OkStatus(); + frame->value_stack().Push(*std::move(result), std::move(attribute_trail)); } class RecursiveImpl : public DirectExpressionStep { From 86954d2771a5399004aa16ca17d7b78936cbcf26 Mon Sep 17 00:00:00 2001 From: Justin King Date: Thu, 1 Oct 2026 14:20:05 -0700 Subject: [PATCH 255/350] Make `ErrorValue` trivial PiperOrigin-RevId: 991900440 --- common/BUILD | 1 + common/legacy_value.cc | 14 +- common/value.cc | 10 +- common/value.h | 3 +- common/value_testing_test.cc | 7 +- common/values/custom_list_value.cc | 4 +- common/values/custom_list_value_test.cc | 5 +- common/values/custom_map_value_test.cc | 14 +- common/values/custom_struct_value_test.cc | 16 +- common/values/error_value.cc | 86 ++--- common/values/error_value.h | 162 +++------ common/values/error_value_test.cc | 19 +- common/values/legacy_list_value.cc | 4 +- common/values/parsed_json_list_value.cc | 4 +- common/values/parsed_json_map_value.cc | 4 +- common/values/parsed_map_field_value.cc | 2 +- common/values/parsed_message_value.cc | 13 +- common/values/parsed_repeated_field_value.cc | 2 +- common/values/string_value.cc | 84 +++-- common/values/string_value.h | 8 +- common/values/string_value_test.cc | 41 ++- common/values/struct_value_builder.cc | 329 +++++++++++------- common/values/value_builder.cc | 12 +- common/values/value_variant.cc | 81 ----- common/values/value_variant.h | 12 +- eval/eval/BUILD | 1 + eval/eval/attribute_utility.cc | 7 +- eval/eval/attribute_utility.h | 3 +- eval/eval/create_list_step.cc | 9 +- eval/eval/create_list_step_test.cc | 8 +- eval/eval/create_map_step.cc | 11 +- eval/eval/create_struct_step.cc | 5 +- eval/eval/equality_steps_test.cc | 2 +- eval/eval/ident_step.cc | 2 +- eval/eval/logic_step.cc | 3 +- eval/eval/logic_step_test.cc | 17 +- eval/eval/optional_or_step_test.cc | 12 +- eval/eval/select_step.cc | 2 +- eval/eval/select_step_test.cc | 4 +- eval/eval/ternary_step_test.cc | 3 +- extensions/comprehensions_v2_functions.cc | 12 +- extensions/math_ext.cc | 4 +- extensions/protobuf/value.h | 5 +- extensions/regex_ext.cc | 8 +- extensions/regex_functions.cc | 6 +- extensions/select_optimization.cc | 2 +- extensions/select_optimization_test.cc | 2 +- extensions/strings.cc | 15 +- runtime/function_adapter_test.cc | 10 +- runtime/internal/BUILD | 1 + runtime/internal/convert_constant.cc | 2 +- runtime/internal/function_adapter_test.cc | 5 +- .../internal/legacy_runtime_type_provider.cc | 6 +- runtime/optional_types_test.cc | 2 +- runtime/standard/BUILD | 1 + runtime/standard/equality_functions.cc | 2 +- runtime/standard/logical_functions_test.cc | 4 +- runtime/standard/regex_functions.cc | 6 +- 58 files changed, 518 insertions(+), 601 deletions(-) diff --git a/common/BUILD b/common/BUILD index ab1d389a8..5b6349624 100644 --- a/common/BUILD +++ b/common/BUILD @@ -589,6 +589,7 @@ cc_test( "//internal:testing", "@com_google_absl//absl/status", "@com_google_absl//absl/time", + "@com_google_protobuf//:protobuf", ], ) diff --git a/common/legacy_value.cc b/common/legacy_value.cc index c6f00d519..d107c6f8a 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -721,7 +721,7 @@ absl::Status LegacyMapValue::Get( CEL_ASSIGN_OR_RETURN(auto cel_key, LegacyValue(arena, key)); auto cel_value = impl_->Get(arena, cel_key); if (!cel_value.has_value()) { - *result = NoSuchKeyError(key.DebugString()); + *result = NoSuchKeyError(key.DebugString(), arena); return absl::OkStatus(); } CEL_RETURN_IF_ERROR(ModernValue(arena, *cel_value, *result)); @@ -928,7 +928,7 @@ absl::Status LegacyStructValue::GetFieldByName( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const { if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { - *result = NoSuchFieldError(name); + *result = NoSuchFieldError(name, arena); return absl::OkStatus(); } @@ -939,7 +939,7 @@ absl::Status LegacyStructValue::GetFieldByName( field = descriptor->file()->pool()->FindExtensionByPrintableName(descriptor, name); if (field == nullptr) { - *result = NoSuchFieldError(name); + *result = NoSuchFieldError(name, arena); return absl::OkStatus(); } } @@ -962,7 +962,7 @@ absl::StatusOr LegacyStructValue::HasFieldByName( absl::string_view name) const { ABSL_DCHECK(message_ptr_ != nullptr); if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); } return UnsafeParsedMessageValue(message_ptr_).HasFieldByName(name); } @@ -970,7 +970,7 @@ absl::StatusOr LegacyStructValue::HasFieldByName( absl::StatusOr LegacyStructValue::HasFieldByNumber(int64_t number) const { ABSL_DCHECK(message_ptr_ != nullptr); if (ABSL_PREDICT_FALSE(legacy_type_info_ == TrivialTypeInfo::GetInstance())) { - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } return UnsafeParsedMessageValue(message_ptr_).HasFieldByNumber(number); } @@ -1008,7 +1008,7 @@ absl::Status LegacyStructValue::Qualify( return field.GetStringKey().value_or(""); }), qualifiers.front()); - *result = NoSuchFieldError(field_name); + *result = NoSuchFieldError(field_name, arena); *count = -1; return absl::OkStatus(); } @@ -1084,7 +1084,7 @@ absl::Status ModernValue(google::protobuf::Arena* arena, return absl::OkStatus(); } case CelValue::Type::kError: - result = ErrorValue{*legacy_value.ErrorOrDie()}; + result = ErrorValue::From(*legacy_value.ErrorOrDie(), arena); return absl::OkStatus(); case CelValue::Type::kAny: return absl::InternalError(absl::StrCat( diff --git a/common/value.cc b/common/value.cc index 1656d4857..cc3a0825f 100644 --- a/common/value.cc +++ b/common/value.cc @@ -200,9 +200,8 @@ absl::Status Value::ConvertToJsonArray( json); }, [](const auto& alternative) -> absl::Status { - return TypeConversionError(alternative.GetTypeName(), - "google.protobuf.ListValue") - .NativeValue(); + return common_internal::MakeTypeConversionError( + alternative.GetTypeName(), "google.protobuf.ListValue"); })); } @@ -257,9 +256,8 @@ absl::Status Value::ConvertToJsonObject( json); }, [](const auto& alternative) -> absl::Status { - return TypeConversionError(alternative.GetTypeName(), - "google.protobuf.Struct") - .NativeValue(); + return common_internal::MakeTypeConversionError( + alternative.GetTypeName(), "google.protobuf.Struct"); })); } diff --git a/common/value.h b/common/value.h index 881f8c925..c2c8a0c7a 100644 --- a/common/value.h +++ b/common/value.h @@ -2612,8 +2612,7 @@ static_assert(std::is_nothrow_swappable_v); inline common_internal::ImplicitlyConvertibleStatus ErrorValueAssign::operator()(absl::Status status) const { - *value_ = arena_ != nullptr ? ErrorValue::From(std::move(status), arena_) - : ErrorValue(std::move(status)); + *value_ = ErrorValue::From(std::move(status), arena_); return common_internal::ImplicitlyConvertibleStatus(); } diff --git a/common/value_testing_test.cc b/common/value_testing_test.cc index 425ce92cc..6aead7ce0 100644 --- a/common/value_testing_test.cc +++ b/common/value_testing_test.cc @@ -21,6 +21,7 @@ #include "absl/time/time.h" #include "common/value.h" #include "internal/testing.h" +#include "google/protobuf/arena.h" namespace cel::test { namespace { @@ -156,12 +157,14 @@ TEST(BytesValueIs, NonMatchMessage) { } TEST(ErrorValueIs, Match) { - EXPECT_THAT(ErrorValue(absl::InternalError("test")), + google::protobuf::Arena arena; + EXPECT_THAT(ErrorValue::From(absl::InternalError("test"), &arena), ErrorValueIs(StatusIs(absl::StatusCode::kInternal, "test"))); } TEST(ErrorValueIs, NoMatch) { - EXPECT_THAT(ErrorValue(absl::UnknownError("test")), + google::protobuf::Arena arena; + EXPECT_THAT(ErrorValue::From(absl::UnknownError("test"), &arena), Not(ErrorValueIs(StatusIs(absl::StatusCode::kInternal, "test")))); EXPECT_THAT(IntValue(2), Not(ErrorValueIs(_))); } diff --git a/common/values/custom_list_value.cc b/common/values/custom_list_value.cc index ca001b103..303f49080 100644 --- a/common/values/custom_list_value.cc +++ b/common/values/custom_list_value.cc @@ -97,9 +97,9 @@ class EmptyListValue final : public common_internal::CompatListValue { private: absl::Status Get(size_t index, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - google::protobuf::Arena* absl_nonnull, + google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const override { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } }; diff --git a/common/values/custom_list_value_test.cc b/common/values/custom_list_value_test.cc index ea0c53d2e..eefc9a481 100644 --- a/common/values/custom_list_value_test.cc +++ b/common/values/custom_list_value_test.cc @@ -24,7 +24,6 @@ #include "absl/status/statusor.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/native_type.h" #include "common/value.h" @@ -121,7 +120,7 @@ class CustomListValueInterfaceTest final : public CustomListValueInterface { *result = IntValue(1); return absl::OkStatus(); } - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } @@ -221,7 +220,7 @@ class CustomListValueTest : public common_internal::ValueTest<> { *result = IntValue(1); return absl::OkStatus(); } - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); }, .clone = [](const CustomListValueDispatcher* absl_nonnull dispatcher, diff --git a/common/values/custom_map_value_test.cc b/common/values/custom_map_value_test.cc index 2eb833ac0..23fc18cf0 100644 --- a/common/values/custom_map_value_test.cc +++ b/common/values/custom_map_value_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -24,7 +25,6 @@ #include "absl/status/statusor.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/memory.h" #include "common/native_type.h" #include "common/value.h" @@ -591,7 +591,8 @@ TEST_F(CustomMapValueTest, Interface_Find_InvalidKeyType) { TEST_F(CustomMapValueTest, Dispatcher_Find_SpecialKeys) { CustomMapValue map = MakeDispatcher(); Value result; - ErrorValue error_key(absl::CancelledError("cancelled")); + ErrorValue error_key = + ErrorValue::From(absl::CancelledError("cancelled"), arena()); ASSERT_THAT(map.Find(error_key, descriptor_pool(), message_factory(), arena(), &result), IsOkAndHolds(false)); @@ -623,7 +624,8 @@ TEST_F(CustomMapValueTest, Dispatcher_Find_SpecialKeys) { TEST_F(CustomMapValueTest, Interface_Find_SpecialKeys) { CustomMapValue map = MakeInterface(); Value result; - ErrorValue error_key(absl::CancelledError("cancelled")); + ErrorValue error_key = + ErrorValue::From(absl::CancelledError("cancelled"), arena()); ASSERT_THAT(map.Find(error_key, descriptor_pool(), message_factory(), arena(), &result), IsOkAndHolds(false)); @@ -735,7 +737,8 @@ TEST_F(CustomMapValueTest, Interface_Has_InvalidKeyType) { TEST_F(CustomMapValueTest, Dispatcher_Has_SpecialKeys) { CustomMapValue map = MakeDispatcher(); Value result; - ErrorValue error_key(absl::CancelledError("cancelled")); + ErrorValue error_key = + ErrorValue::From(absl::CancelledError("cancelled"), arena()); ASSERT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena(), &result), IsOk()); @@ -758,7 +761,8 @@ TEST_F(CustomMapValueTest, Dispatcher_Has_SpecialKeys) { TEST_F(CustomMapValueTest, Interface_Has_SpecialKeys) { CustomMapValue map = MakeInterface(); Value result; - ErrorValue error_key(absl::CancelledError("cancelled")); + ErrorValue error_key = + ErrorValue::From(absl::CancelledError("cancelled"), arena()); ASSERT_THAT(map.Has(error_key, descriptor_pool(), message_factory(), arena(), &result), IsOk()); diff --git a/common/values/custom_struct_value_test.cc b/common/values/custom_struct_value_test.cc index 32d867a4d..c4a34db72 100644 --- a/common/values/custom_struct_value_test.cc +++ b/common/values/custom_struct_value_test.cc @@ -117,7 +117,7 @@ class CustomStructValueInterfaceTest final : public CustomStructValueInterface { *result = IntValue(1); return absl::OkStatus(); } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); } absl::Status GetFieldByNumber( @@ -134,7 +134,7 @@ class CustomStructValueInterfaceTest final : public CustomStructValueInterface { *result = IntValue(1); return absl::OkStatus(); } - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } absl::StatusOr HasFieldByName(absl::string_view name) const override { @@ -144,7 +144,7 @@ class CustomStructValueInterfaceTest final : public CustomStructValueInterface { if (name == "bar") { return true; } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); } absl::StatusOr HasFieldByNumber(int64_t number) const override { @@ -154,7 +154,7 @@ class CustomStructValueInterfaceTest final : public CustomStructValueInterface { if (number == 2) { return true; } - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } absl::Status ForEachField( @@ -283,7 +283,7 @@ class CustomStructValueTest : public common_internal::ValueTest<> { *result = IntValue(1); return absl::OkStatus(); } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); }, .get_field_by_number = [](const CustomStructValueDispatcher* absl_nonnull dispatcher, @@ -301,7 +301,7 @@ class CustomStructValueTest : public common_internal::ValueTest<> { *result = IntValue(1); return absl::OkStatus(); } - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); }, .has_field_by_name = [](const CustomStructValueDispatcher* absl_nonnull dispatcher, @@ -313,7 +313,7 @@ class CustomStructValueTest : public common_internal::ValueTest<> { if (name == "bar") { return true; } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); }, .has_field_by_number = [](const CustomStructValueDispatcher* absl_nonnull dispatcher, @@ -325,7 +325,7 @@ class CustomStructValueTest : public common_internal::ValueTest<> { if (number == 2) { return true; } - return NoSuchFieldError(absl::StrCat(number)).ToStatus(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); }, .for_each_field = [](const CustomStructValueDispatcher* absl_nonnull dispatcher, diff --git a/common/values/error_value.cc b/common/values/error_value.cc index 538ac4234..030bb03b1 100644 --- a/common/values/error_value.cc +++ b/common/values/error_value.cc @@ -13,9 +13,7 @@ // limitations under the License. #include -#include #include -#include #include "absl/base/no_destructor.h" #include "absl/base/nullability.h" @@ -77,30 +75,47 @@ absl::Status MakeIndexOutOfBoundsError(ptrdiff_t index) { } // namespace -ErrorValue::ErrorValue() : ErrorValue(nullptr, &DefaultErrorValue()) {} +namespace common_internal { + +absl::Status MakeTypeConversionError(const Type& from, const Type& to) { + return cel::MakeTypeConversionError(from.DebugString(), to.DebugString()); +} + +absl::Status MakeTypeConversionError(absl::string_view from, + absl::string_view to) { + return cel::MakeTypeConversionError(from, to); +} + +absl::Status MakeNoSuchFieldError(absl::string_view field) { + return cel::MakeNoSuchFieldError(field); +} + +absl::Status MakeNoSuchKeyError(absl::string_view key) { + return cel::MakeNoSuchKeyError(key); +} + +absl::Status MakeIndexOutOfBoundsError(size_t index) { + return cel::MakeIndexOutOfBoundsError(index); +} -ErrorValue NoSuchFieldError(absl::string_view field) { - return ErrorValue(MakeNoSuchFieldError(field)); +absl::Status MakeIndexOutOfBoundsError(ptrdiff_t index) { + return cel::MakeIndexOutOfBoundsError(index); } +} // namespace common_internal + +ErrorValue::ErrorValue() : ErrorValue(nullptr, &DefaultErrorValue()) {} + ErrorValue NoSuchFieldError(absl::string_view field, google::protobuf::Arena* absl_nonnull arena) { return ErrorValue::From(MakeNoSuchFieldError(field), arena); } -ErrorValue NoSuchKeyError(absl::string_view key) { - return ErrorValue(MakeNoSuchKeyError(key)); -} - ErrorValue NoSuchKeyError(absl::string_view key, google::protobuf::Arena* absl_nonnull arena) { return ErrorValue::From(MakeNoSuchKeyError(key), arena); } -ErrorValue NoSuchTypeError(absl::string_view type) { - return ErrorValue(MakeNoSuchTypeError(type)); -} - ErrorValue NoSuchTypeError(absl::string_view type, google::protobuf::Arena* absl_nonnull arena) { return ErrorValue::From(MakeNoSuchTypeError(type), arena); @@ -112,37 +127,21 @@ ErrorValue DuplicateKeyError() { return ErrorValue(nullptr, &*error); } -ErrorValue TypeConversionError(absl::string_view from, absl::string_view to) { - return ErrorValue(MakeTypeConversionError(from, to)); -} - ErrorValue TypeConversionError(absl::string_view from, absl::string_view to, google::protobuf::Arena* absl_nonnull arena) { return ErrorValue::From(MakeTypeConversionError(from, to), arena); } -ErrorValue TypeConversionError(const Type& from, const Type& to) { - return TypeConversionError(from.DebugString(), to.DebugString()); -} - ErrorValue TypeConversionError(const Type& from, const Type& to, google::protobuf::Arena* absl_nonnull arena) { return TypeConversionError(from.DebugString(), to.DebugString(), arena); } -ErrorValue IndexOutOfBoundsError(size_t index) { - return ErrorValue(MakeIndexOutOfBoundsError(index)); -} - ErrorValue IndexOutOfBoundsError(size_t index, google::protobuf::Arena* absl_nonnull arena) { return ErrorValue::From(MakeIndexOutOfBoundsError(index), arena); } -ErrorValue IndexOutOfBoundsError(ptrdiff_t index) { - return ErrorValue(MakeIndexOutOfBoundsError(index)); -} - ErrorValue IndexOutOfBoundsError(ptrdiff_t index, google::protobuf::Arena* absl_nonnull arena) { return ErrorValue::From(MakeIndexOutOfBoundsError(index), arena); @@ -217,36 +216,9 @@ ErrorValue ErrorValue::Clone(google::protobuf::Arena* absl_nonnull arena) const return *this; } -absl::Status ErrorValue::ToStatus() const& { +absl::Status ErrorValue::ToStatus() const { ABSL_DCHECK(*this); - if (status_ptr_ == nullptr) { - return *std::launder( - reinterpret_cast(&status_val_[0])); - } return *status_ptr_; } -absl::Status ErrorValue::ToStatus() && { - ABSL_DCHECK(*this); - if (status_ptr_ == nullptr) { - return std::move( - *std::launder(reinterpret_cast(&status_val_[0]))); - } - return *status_ptr_; -} - -ErrorValue::operator bool() const { - if (status_ptr_ == nullptr) { - return !std::launder(reinterpret_cast(&status_val_[0])) - ->ok(); - } - return !status_ptr_->ok(); -} - -void swap(ErrorValue& lhs, ErrorValue& rhs) noexcept { - ErrorValue tmp(std::move(lhs)); - lhs = std::move(rhs); - rhs = std::move(tmp); -} - } // namespace cel diff --git a/common/values/error_value.h b/common/values/error_value.h index ea96bc857..a3cb2c3b4 100644 --- a/common/values/error_value.h +++ b/common/values/error_value.h @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -32,7 +31,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "common/arena.h" #include "common/type.h" #include "common/value_kind.h" #include "common/values/values.h" @@ -48,6 +46,32 @@ class ErrorValue; ErrorValue DuplicateKeyError(); +namespace common_internal { +absl::Status MakeTypeConversionError(const Type& from, const Type& to); +absl::Status MakeTypeConversionError(absl::string_view from, + absl::string_view to); +absl::Status MakeNoSuchFieldError(absl::string_view field); +absl::Status MakeNoSuchKeyError(absl::string_view key); +absl::Status MakeIndexOutOfBoundsError(size_t index); +absl::Status MakeIndexOutOfBoundsError(ptrdiff_t index); +template +std::enable_if_t, std::is_unsigned, + std::negation>>, + absl::Status> +MakeIndexOutOfBoundsError(T index) { + static_assert(sizeof(T) <= sizeof(size_t)); + return MakeIndexOutOfBoundsError(static_cast(index)); +} +template +std::enable_if_t, std::is_signed, + std::negation>>, + absl::Status> +MakeIndexOutOfBoundsError(T index) { + static_assert(sizeof(T) <= sizeof(ptrdiff_t)); + return MakeIndexOutOfBoundsError(static_cast(index)); +} +} // namespace common_internal + // `ErrorValue` represents values of the `ErrorType`. class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final : private common_internal::ValueMixin { @@ -72,38 +96,12 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final return ErrorValue(nullptr, value); } - ABSL_DEPRECATED("Use From") - explicit ErrorValue(absl::Status value) - : arena_(nullptr), status_ptr_(nullptr) { - ::new (static_cast(&status_val_[0])) absl::Status(std::move(value)); - ABSL_DCHECK(*this) << "ErrorValue requires a non-OK absl::Status"; - } - // By default, this creates an UNKNOWN error. You should always create a more // specific error value. ErrorValue(); - ErrorValue(const ErrorValue& other) { CopyConstruct(other); } - - ErrorValue(ErrorValue&& other) noexcept { MoveConstruct(other); } - - ~ErrorValue() { Destruct(); } - - ErrorValue& operator=(const ErrorValue& other) { - if (this != &other) { - Destruct(); - CopyConstruct(other); - } - return *this; - } - - ErrorValue& operator=(ErrorValue&& other) noexcept { - if (this != &other) { - Destruct(); - MoveConstruct(other); - } - return *this; - } + ErrorValue(const ErrorValue&) = default; + ErrorValue& operator=(const ErrorValue&) = default; static constexpr ValueKind kind() { return kKind; } @@ -134,9 +132,7 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final ErrorValue Clone(google::protobuf::Arena* absl_nonnull arena) const; - absl::Status ToStatus() const&; - - absl::Status ToStatus() &&; + absl::Status ToStatus() const; ABSL_DEPRECATED("Use ToStatus()") absl::Status NativeValue() const& { return ToStatus(); } @@ -144,111 +140,58 @@ class ABSL_ATTRIBUTE_TRIVIAL_ABI ErrorValue final ABSL_DEPRECATED("Use ToStatus()") absl::Status NativeValue() && { return std::move(*this).ToStatus(); } - friend void swap(ErrorValue& lhs, ErrorValue& rhs) noexcept; + friend void swap(ErrorValue& lhs, ErrorValue& rhs) noexcept { + using std::swap; + swap(lhs.arena_, rhs.arena_); + swap(lhs.status_ptr_, rhs.status_ptr_); + } - explicit operator bool() const; + explicit operator bool() const { return !status_ptr_->ok(); } private: friend ErrorValue DuplicateKeyError(); friend class common_internal::ValueMixin; - friend struct ArenaTraits; ErrorValue(google::protobuf::Arena* absl_nullable arena, const absl::Status* absl_nonnull status) : arena_(arena), status_ptr_(status) {} - void CopyConstruct(const ErrorValue& other) { - arena_ = other.arena_; - status_ptr_ = other.status_ptr_; - if (status_ptr_ == nullptr) { - ::new (static_cast(&status_val_[0])) absl::Status(*std::launder( - reinterpret_cast(&other.status_val_[0]))); - } - } - - void MoveConstruct(ErrorValue& other) { - arena_ = other.arena_; - status_ptr_ = other.status_ptr_; - if (status_ptr_ == nullptr) { - ::new (static_cast(&status_val_[0])) - absl::Status(std::move(*std::launder( - reinterpret_cast(&other.status_val_[0])))); - } - } - - void Destruct() { - if (status_ptr_ == nullptr) { - std::launder(reinterpret_cast(&status_val_[0]))->~Status(); - } - } - google::protobuf::Arena* absl_nullable arena_; - const absl::Status* absl_nullable status_ptr_ = nullptr; - alignas(absl::Status) char status_val_[sizeof(absl::Status)]; + const absl::Status* absl_nonnull status_ptr_; }; -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -ErrorValue NoSuchFieldError(absl::string_view field); ErrorValue NoSuchFieldError(absl::string_view field, google::protobuf::Arena* absl_nonnull arena); -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -ErrorValue NoSuchKeyError(absl::string_view key); ErrorValue NoSuchKeyError(absl::string_view key, google::protobuf::Arena* absl_nonnull arena); -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -ErrorValue NoSuchTypeError(absl::string_view type); ErrorValue NoSuchTypeError(absl::string_view type, google::protobuf::Arena* absl_nonnull arena); ErrorValue DuplicateKeyError(); -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -ErrorValue TypeConversionError(absl::string_view from, absl::string_view to); ErrorValue TypeConversionError(absl::string_view from, absl::string_view to, google::protobuf::Arena* absl_nonnull arena); -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -ErrorValue TypeConversionError(const Type& from, const Type& to); ErrorValue TypeConversionError(const Type& from, const Type& to, google::protobuf::Arena* absl_nonnull arena); -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -ErrorValue IndexOutOfBoundsError(size_t index); ErrorValue IndexOutOfBoundsError(size_t index, google::protobuf::Arena* absl_nonnull arena); -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -ErrorValue IndexOutOfBoundsError(ptrdiff_t index); ErrorValue IndexOutOfBoundsError(ptrdiff_t index, google::protobuf::Arena* absl_nonnull arena); // Catch other integrals and forward them to the above ones. This is needed to // avoid ambiguous overload issues for smaller integral types like `int`. template -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -std::enable_if_t, std::is_unsigned, - std::negation>>, - ErrorValue> IndexOutOfBoundsError(T index) { - static_assert(sizeof(T) <= sizeof(size_t)); - return IndexOutOfBoundsError(static_cast(index)); -} -template std::enable_if_t, std::is_unsigned, std::negation>>, ErrorValue> IndexOutOfBoundsError(T index, google::protobuf::Arena* absl_nonnull arena) { static_assert(sizeof(T) <= sizeof(size_t)); - return IndexOutOfBoundsError(static_cast(index)); -} -template -ABSL_DEPRECATED("Use the overload which takes google::protobuf::Arena*") -std::enable_if_t, std::is_signed, - std::negation>>, - ErrorValue> IndexOutOfBoundsError(T index) { - static_assert(sizeof(T) <= sizeof(ptrdiff_t)); - return IndexOutOfBoundsError(static_cast(index)); + return IndexOutOfBoundsError(static_cast(index), arena); } template std::enable_if_t, std::is_signed, @@ -256,7 +199,7 @@ std::enable_if_t, std::is_signed, ErrorValue> IndexOutOfBoundsError(T index, google::protobuf::Arena* absl_nonnull arena) { static_assert(sizeof(T) <= sizeof(ptrdiff_t)); - return IndexOutOfBoundsError(static_cast(index)); + return IndexOutOfBoundsError(static_cast(index), arena); } inline std::ostream& operator<<(std::ostream& out, const ErrorValue& value) { @@ -269,16 +212,12 @@ bool IsNoSuchKey(const ErrorValue& value); class ErrorValueReturn final { public: - ABSL_DEPRECATED("Use constructor which takes google::protobuf::Arena*") - ErrorValueReturn() = default; - explicit ErrorValueReturn(google::protobuf::Arena* absl_nonnull arena) : arena_(arena) { ABSL_DCHECK(arena != nullptr); } ErrorValue operator()(absl::Status status) const { - return arena_ != nullptr ? ErrorValue::From(std::move(status), arena_) - : ErrorValue(std::move(status)); + return ErrorValue::From(std::move(status), arena_); } private: @@ -300,8 +239,9 @@ struct ImplicitlyConvertibleStatus { } // namespace common_internal -// For use with `RETURN_IF_ERROR(...).With(cel::ErrorValueAssign(&result))` and -// `ASSIGN_OR_RETURN(..., ..., _.With(cel::ErrorValueAssign(&result)))`. +// For use with `RETURN_IF_ERROR(...).With(cel::ErrorValueAssign(&result, +// arena))` and `ASSIGN_OR_RETURN(..., ..., +// _.With(cel::ErrorValueAssign(&result, arena)))`. // // IMPORTANT: // If the returning type is `absl::Status` the result will be @@ -311,17 +251,6 @@ class ErrorValueAssign final { public: ErrorValueAssign() = delete; - ABSL_DEPRECATED("Use constructor which takes google::protobuf::Arena*") - explicit ErrorValueAssign(Value& value ABSL_ATTRIBUTE_LIFETIME_BOUND) - : ErrorValueAssign(std::addressof(value)) {} - - ABSL_DEPRECATED("Use constructor which takes google::protobuf::Arena*") - explicit ErrorValueAssign( - Value* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND) - : value_(value) { - ABSL_DCHECK(value != nullptr); - } - ErrorValueAssign(Value& value ABSL_ATTRIBUTE_LIFETIME_BOUND, google::protobuf::Arena* absl_nonnull arena) : ErrorValueAssign(std::addressof(value), arena) {} @@ -341,13 +270,6 @@ class ErrorValueAssign final { google::protobuf::Arena* arena_ = nullptr; }; -template <> -struct ArenaTraits { - static bool trivially_destructible(const ErrorValue& value) { - return value.status_ptr_ != nullptr; - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_ERROR_VALUE_H_ diff --git a/common/values/error_value_test.cc b/common/values/error_value_test.cc index 343a93d19..288300f89 100644 --- a/common/values/error_value_test.cc +++ b/common/values/error_value_test.cc @@ -33,28 +33,30 @@ using ErrorValueTest = common_internal::ValueTest<>; TEST_F(ErrorValueTest, Default) { ErrorValue value; - EXPECT_THAT(value.NativeValue(), StatusIs(absl::StatusCode::kUnknown)); + EXPECT_THAT(value.ToStatus(), StatusIs(absl::StatusCode::kUnknown)); } TEST_F(ErrorValueTest, OkStatus) { - EXPECT_DEBUG_DEATH(static_cast(ErrorValue(absl::OkStatus())), _); + EXPECT_DEBUG_DEATH( + static_cast(ErrorValue::From(absl::OkStatus(), arena())), _); } TEST_F(ErrorValueTest, Kind) { - EXPECT_EQ(ErrorValue(absl::CancelledError()).kind(), ErrorValue::kKind); - EXPECT_EQ(Value(ErrorValue(absl::CancelledError())).kind(), + EXPECT_EQ(ErrorValue::From(absl::CancelledError(), arena()).kind(), + ErrorValue::kKind); + EXPECT_EQ(Value(ErrorValue::From(absl::CancelledError(), arena())).kind(), ErrorValue::kKind); } TEST_F(ErrorValueTest, DebugString) { { std::ostringstream out; - out << ErrorValue(absl::CancelledError()); + out << ErrorValue::From(absl::CancelledError(), arena()); EXPECT_THAT(out.str(), Not(IsEmpty())); } { std::ostringstream out; - out << Value(ErrorValue(absl::CancelledError())); + out << Value(ErrorValue::From(absl::CancelledError(), arena())); EXPECT_THAT(out.str(), Not(IsEmpty())); } } @@ -74,9 +76,10 @@ TEST_F(ErrorValueTest, ConvertToJson) { } TEST_F(ErrorValueTest, NativeTypeId) { - EXPECT_EQ(NativeTypeId::Of(ErrorValue(absl::CancelledError())), + EXPECT_EQ(NativeTypeId::Of(ErrorValue::From(absl::CancelledError(), arena())), NativeTypeId::For()); - EXPECT_EQ(NativeTypeId::Of(Value(ErrorValue(absl::CancelledError()))), + EXPECT_EQ(NativeTypeId::Of( + Value(ErrorValue::From(absl::CancelledError(), arena()))), NativeTypeId::For()); } diff --git a/common/values/legacy_list_value.cc b/common/values/legacy_list_value.cc index 1152df715..d3c1b94be 100644 --- a/common/values/legacy_list_value.cc +++ b/common/values/legacy_list_value.cc @@ -67,7 +67,7 @@ class LegacyParsedRepeatedFieldListValue final if (ABSL_PREDICT_FALSE(index < 0 || index >= size())) { return google::api::expr::runtime::CelValue::CreateError( google::protobuf::Arena::Create( - arena, IndexOutOfBoundsError(index).ToStatus())); + arena, common_internal::MakeIndexOutOfBoundsError(index))); } Value result; auto status = value_.Get( @@ -189,7 +189,7 @@ class LegacyParsedJsonListValue final if (ABSL_PREDICT_FALSE(index < 0 || index >= size())) { return google::api::expr::runtime::CelValue::CreateError( google::protobuf::Arena::Create( - arena, IndexOutOfBoundsError(index).ToStatus())); + arena, common_internal::MakeIndexOutOfBoundsError(index))); } Value result; auto status = value_.Get( diff --git a/common/values/parsed_json_list_value.cc b/common/values/parsed_json_list_value.cc index 936853f23..9418aebad 100644 --- a/common/values/parsed_json_list_value.cc +++ b/common/values/parsed_json_list_value.cc @@ -220,14 +220,14 @@ absl::Status ParsedJsonListValue::Get( ABSL_DCHECK(result != nullptr); if (value_ == nullptr) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } const auto reflection = well_known_types::GetListValueReflectionOrDie(value_->GetDescriptor()); if (ABSL_PREDICT_FALSE(index >= static_cast(reflection.ValuesSize(*value_)))) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } *result = common_internal::ParsedJsonValue( diff --git a/common/values/parsed_json_map_value.cc b/common/values/parsed_json_map_value.cc index ad0ad558c..1b568e046 100644 --- a/common/values/parsed_json_map_value.cc +++ b/common/values/parsed_json_map_value.cc @@ -28,7 +28,6 @@ #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "common/allocator.h" #include "common/memory.h" #include "common/value.h" #include "common/values/parsed_json_value.h" @@ -40,7 +39,6 @@ #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/io/zero_copy_stream.h" -#include "google/protobuf/map.h" #include "google/protobuf/map_field.h" #include "google/protobuf/message.h" #include "google/protobuf/message_lite.h" @@ -218,7 +216,7 @@ absl::Status ParsedJsonMapValue::Get( CEL_ASSIGN_OR_RETURN( bool ok, Find(key, descriptor_pool, message_factory, arena, result)); if (ABSL_PREDICT_FALSE(!ok) && !(result->IsError() || result->IsUnknown())) { - *result = NoSuchKeyError(key.DebugString()); + *result = NoSuchKeyError(key.DebugString(), arena); } return absl::OkStatus(); } diff --git a/common/values/parsed_map_field_value.cc b/common/values/parsed_map_field_value.cc index 8dd7c3d02..e17bfb4d2 100644 --- a/common/values/parsed_map_field_value.cc +++ b/common/values/parsed_map_field_value.cc @@ -326,7 +326,7 @@ absl::Status ParsedMapFieldValue::Get( CEL_ASSIGN_OR_RETURN( bool ok, Find(key, descriptor_pool, message_factory, arena, result)); if (ABSL_PREDICT_FALSE(!ok) && !(result->IsError() || result->IsUnknown())) { - *result = ErrorValue(NoSuchKeyError(key.DebugString())); + *result = NoSuchKeyError(key.DebugString(), arena); } return absl::OkStatus(); } diff --git a/common/values/parsed_message_value.cc b/common/values/parsed_message_value.cc index a7cc01e42..15ce7a4cb 100644 --- a/common/values/parsed_message_value.cc +++ b/common/values/parsed_message_value.cc @@ -40,7 +40,6 @@ #include "internal/json.h" #include "internal/message_equality.h" #include "internal/status_macros.h" -#include "internal/well_known_types.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -185,7 +184,7 @@ absl::Status ParsedMessageValue::GetFieldByName( field = descriptor->file()->pool()->FindExtensionByPrintableName(descriptor, name); if (field == nullptr) { - *result = NoSuchFieldError(name); + *result = NoSuchFieldError(name, arena); return absl::OkStatus(); } } @@ -206,12 +205,12 @@ absl::Status ParsedMessageValue::GetFieldByNumber( const auto* descriptor = GetDescriptor(); if (number < std::numeric_limits::min() || number > std::numeric_limits::max()) { - *result = NoSuchFieldError(absl::StrCat(number)); + *result = NoSuchFieldError(absl::StrCat(number), arena); return absl::OkStatus(); } const auto* field = descriptor->FindFieldByNumber(static_cast(number)); if (field == nullptr) { - *result = NoSuchFieldError(absl::StrCat(number)); + *result = NoSuchFieldError(absl::StrCat(number), arena); return absl::OkStatus(); } return GetField(field, unboxing_options, descriptor_pool, message_factory, @@ -226,7 +225,7 @@ absl::StatusOr ParsedMessageValue::HasFieldByName( field = descriptor->file()->pool()->FindExtensionByPrintableName(descriptor, name); if (field == nullptr) { - return NoSuchFieldError(name).NativeValue(); + return common_internal::MakeNoSuchFieldError(name); } } return HasField(field); @@ -237,11 +236,11 @@ absl::StatusOr ParsedMessageValue::HasFieldByNumber( const auto* descriptor = GetDescriptor(); if (number < std::numeric_limits::min() || number > std::numeric_limits::max()) { - return NoSuchFieldError(absl::StrCat(number)).NativeValue(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } const auto* field = descriptor->FindFieldByNumber(static_cast(number)); if (field == nullptr) { - return NoSuchFieldError(absl::StrCat(number)).NativeValue(); + return common_internal::MakeNoSuchFieldError(absl::StrCat(number)); } return HasField(field); } diff --git a/common/values/parsed_repeated_field_value.cc b/common/values/parsed_repeated_field_value.cc index db9a810ff..b716962cc 100644 --- a/common/values/parsed_repeated_field_value.cc +++ b/common/values/parsed_repeated_field_value.cc @@ -246,7 +246,7 @@ absl::Status ParsedRepeatedFieldValue::Get( index >= std::numeric_limits::max() || static_cast(index) >= GetReflection()->FieldSize(*message_, field_))) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } if (arena_ == nullptr) { diff --git a/common/values/string_value.cc b/common/values/string_value.cc index 7659e9a71..053a456bd 100644 --- a/common/values/string_value.cc +++ b/common/values/string_value.cc @@ -642,15 +642,20 @@ absl::StatusOr SubstringImpl(const absl::Cord& cord, uint64_t start) { } // namespace -Value StringValue::Substring(int64_t start) const { +Value StringValue::Substring(int64_t start, + google::protobuf::Arena* absl_nonnull arena) const { if (start < 0) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(): is less than 0")); + return ErrorValue::From( + absl::InvalidArgumentError( + ".substring(): is less than 0"), + arena); } if (static_cast(start) > value_.size()) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(, ): or is greater than " - ".size()")); + return ErrorValue::From( + absl::InvalidArgumentError(".substring(, ): " + " or is greater than " + ".size()"), + arena); } if (start == 0) { return *this; @@ -660,7 +665,7 @@ Value StringValue::Substring(int64_t start) const { absl::StatusOr status_or_index = (SubstringImpl)(value_.GetSmall(), start); if (!status_or_index.ok()) { - return ErrorValue(std::move(status_or_index).status()); + return ErrorValue::From(std::move(status_or_index).status(), arena); } StringValue result; result.value_.rep_.header.kind = common_internal::ByteStringKind::kSmall; @@ -675,7 +680,7 @@ Value StringValue::Substring(int64_t start) const { absl::StatusOr status_or_index = (SubstringImpl)(value_.GetMedium(), start); if (!status_or_index.ok()) { - return ErrorValue(std::move(status_or_index).status()); + return ErrorValue::From(std::move(status_or_index).status(), arena); } StringValue result; result.value_.rep_.header.kind = common_internal::ByteStringKind::kMedium; @@ -690,7 +695,7 @@ Value StringValue::Substring(int64_t start) const { absl::StatusOr status_or_index = (SubstringImpl)(value_.GetLarge(), start); if (!status_or_index.ok()) { - return ErrorValue(std::move(status_or_index).status()); + return ErrorValue::From(std::move(status_or_index).status(), arena); } return StringValue(common_internal::ByteString::Wrap( value_.rep_.large.data, value_.rep_.large.offset + *status_or_index, @@ -760,27 +765,34 @@ absl::StatusOr> SubstringImpl(const absl::Cord& cord, } // namespace -Value StringValue::Substring(int64_t start, int64_t end) const { +Value StringValue::Substring(int64_t start, int64_t end, + google::protobuf::Arena* absl_nonnull arena) const { if (start < 0) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(, ): is less than 0")); + return ErrorValue::From( + absl::InvalidArgumentError( + ".substring(, ): is less than 0"), + arena); } if (end < start) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(, ): is less than ")); + return ErrorValue::From( + absl::InvalidArgumentError( + ".substring(, ): is less than "), + arena); } if (static_cast(start) > value_.size() || static_cast(end) > value_.size()) { - return ErrorValue(absl::InvalidArgumentError( - ".substring(, ): or is greater than " - ".size()")); + return ErrorValue::From( + absl::InvalidArgumentError(".substring(, ): " + " or is greater than " + ".size()"), + arena); } switch (value_.GetKind()) { case common_internal::ByteStringKind::kSmall: { absl::StatusOr> status_or_indices = (SubstringImpl)(value_.GetSmall(), start, end); if (!status_or_indices.ok()) { - return ErrorValue(std::move(status_or_indices).status()); + return ErrorValue::From(std::move(status_or_indices).status(), arena); } StringValue result; result.value_.rep_.header.kind = common_internal::ByteStringKind::kSmall; @@ -796,7 +808,7 @@ Value StringValue::Substring(int64_t start, int64_t end) const { absl::StatusOr> status_or_indices = (SubstringImpl)(value_.GetMedium(), start, end); if (!status_or_indices.ok()) { - return ErrorValue(std::move(status_or_indices).status()); + return ErrorValue::From(std::move(status_or_indices).status(), arena); } StringValue result; result.value_.rep_.header.kind = common_internal::ByteStringKind::kMedium; @@ -811,7 +823,7 @@ Value StringValue::Substring(int64_t start, int64_t end) const { absl::StatusOr> status_or_indices = (SubstringImpl)(value_.GetLarge(), start, end); if (!status_or_indices.ok()) { - return ErrorValue(std::move(status_or_indices).status()); + return ErrorValue::From(std::move(status_or_indices).status(), arena); } return StringValue(common_internal::ByteString::Wrap( value_.rep_.large.data, @@ -1243,8 +1255,8 @@ absl::Status StringValue::Join( string_element->AppendToString(&joined); } else { ABSL_DCHECK(!element->Is()); - *result = - ErrorValue(runtime_internal::CreateNoMatchingOverloadError("join")); + *result = ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("join"), arena); return absl::OkStatus(); } while (true) { @@ -1258,8 +1270,8 @@ absl::Status StringValue::Join( string_element->AppendToString(&joined); } else { ABSL_DCHECK(!element->Is()); - *result = - ErrorValue(runtime_internal::CreateNoMatchingOverloadError("join")); + *result = ErrorValue::From( + runtime_internal::CreateNoMatchingOverloadError("join"), arena); return absl::OkStatus(); } } @@ -1456,13 +1468,15 @@ absl::Status StringValue::Replace(const StringValue& needle, return absl::OkStatus(); } -Value StringValue::CharAt(int64_t pos) const { +Value StringValue::CharAt(int64_t pos, + google::protobuf::Arena* absl_nonnull arena) const { if (pos < 0) { - return ErrorValue(absl::InvalidArgumentError( - ".charAt(): is less than 0")); + return ErrorValue::From(absl::InvalidArgumentError( + ".charAt(): is less than 0"), + arena); } return value_.Visit(absl::Overload( - [this, pos](absl::string_view rep) mutable -> Value { + [this, pos, arena](absl::string_view rep) mutable -> Value { while (!rep.empty()) { char32_t code_point; size_t code_units; @@ -1485,10 +1499,12 @@ Value StringValue::CharAt(int64_t pos) const { if (pos == 0) { return StringValue(); } - return ErrorValue(absl::InvalidArgumentError( - ".charAt(): is greater than .size()")); + return ErrorValue::From( + absl::InvalidArgumentError(".charAt(): is " + "greater than .size()"), + arena); }, - [pos](const absl::Cord& rep) mutable -> Value { + [pos, arena](const absl::Cord& rep) mutable -> Value { absl::Cord::CharIterator begin = rep.char_begin(); absl::Cord::CharIterator end = rep.char_end(); while (begin != end) { @@ -1513,8 +1529,10 @@ Value StringValue::CharAt(int64_t pos) const { if (pos == 0) { return StringValue(); } - return ErrorValue(absl::InvalidArgumentError( - ".charAt(): is greater than .size()")); + return ErrorValue::From( + absl::InvalidArgumentError(".charAt(): is " + "greater than .size()"), + arena); })); } diff --git a/common/values/string_value.h b/common/values/string_value.h index aa478ce84..98ca43234 100644 --- a/common/values/string_value.h +++ b/common/values/string_value.h @@ -26,7 +26,6 @@ #include #include "absl/base/attributes.h" -#include "absl/base/macros.h" #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -258,9 +257,10 @@ class StringValue final : private common_internal::ValueMixin { absl::optional LastIndexOf(const StringValue& string, int64_t pos) const; - Value Substring(int64_t start) const; + Value Substring(int64_t start, google::protobuf::Arena* absl_nonnull arena) const; - Value Substring(int64_t start, int64_t end) const; + Value Substring(int64_t start, int64_t end, + google::protobuf::Arena* absl_nonnull arena) const; // Returns a new `StringValue` with all lowercase ASCII characters // converted to lowercase. @@ -325,7 +325,7 @@ class StringValue final : private common_internal::ValueMixin { // Returns the character at `pos` as a new `StringValue`. `pos` is a // 0-based index based on Unicode code points. Returns `ErrorValue` if `pos` // is out of range. - Value CharAt(int64_t pos) const; + Value CharAt(int64_t pos, google::protobuf::Arena* absl_nonnull arena) const; absl::optional TryFlat() const ABSL_ATTRIBUTE_LIFETIME_BOUND { diff --git a/common/values/string_value_test.cc b/common/values/string_value_test.cc index be21e4d1e..6bfcb83aa 100644 --- a/common/values/string_value_test.cc +++ b/common/values/string_value_test.cc @@ -22,7 +22,6 @@ #include "absl/strings/cord.h" #include "absl/strings/cord_test_helpers.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "common/native_type.h" #include "common/value.h" #include "common/value_testing.h" @@ -436,25 +435,25 @@ TEST_F(StringValueTest, CharAt) { StringValue unicode_string_cord = StringValue::From(absl::Cord("aμc"), arena()); - EXPECT_THAT(big_string.CharAt(0), StringValueIs("T")); - EXPECT_THAT(big_string_cord.CharAt(0), StringValueIs("T")); - EXPECT_THAT(small_string.CharAt(1), StringValueIs("b")); - EXPECT_THAT(small_string_cord.CharAt(1), StringValueIs("b")); - EXPECT_THAT(unicode_string.CharAt(1), StringValueIs("μ")); - EXPECT_THAT(unicode_string_cord.CharAt(1), StringValueIs("μ")); + EXPECT_THAT(big_string.CharAt(0, arena()), StringValueIs("T")); + EXPECT_THAT(big_string_cord.CharAt(0, arena()), StringValueIs("T")); + EXPECT_THAT(small_string.CharAt(1, arena()), StringValueIs("b")); + EXPECT_THAT(small_string_cord.CharAt(1, arena()), StringValueIs("b")); + EXPECT_THAT(unicode_string.CharAt(1, arena()), StringValueIs("μ")); + EXPECT_THAT(unicode_string_cord.CharAt(1, arena()), StringValueIs("μ")); EXPECT_THAT( - big_string.CharAt(100), + big_string.CharAt(100, arena()), ErrorValueIs(absl::InvalidArgumentError( ".charAt(): is greater than .size()"))); EXPECT_THAT( - big_string_cord.CharAt(100), + big_string_cord.CharAt(100, arena()), ErrorValueIs(absl::InvalidArgumentError( ".charAt(): is greater than .size()"))); - EXPECT_THAT(big_string.CharAt(-1), + EXPECT_THAT(big_string.CharAt(-1, arena()), ErrorValueIs(absl::InvalidArgumentError( ".charAt(): is less than 0"))); - EXPECT_THAT(big_string_cord.CharAt(-1), + EXPECT_THAT(big_string_cord.CharAt(-1, arena()), ErrorValueIs(absl::InvalidArgumentError( ".charAt(): is less than 0"))); } @@ -470,20 +469,20 @@ TEST_F(StringValueTest, Substring) { StringValue unicode_cord = StringValue::From(absl::Cord("€€€€€€"), arena()); StringValue unicode_view = StringValue::WrapUnsafe("€€€€€€"); - EXPECT_THAT(unicode_cord.Substring(0, 2), StringValueIs("€€")); - EXPECT_THAT(unicode_view.Substring(0, 2), StringValueIs("€€")); - EXPECT_THAT(unicode_cord.Substring(1, 2), StringValueIs("€")); - EXPECT_THAT(unicode_view.Substring(1, 2), StringValueIs("€")); - EXPECT_THAT(unicode_cord.Substring(2, 4), StringValueIs("€€")); - EXPECT_THAT(unicode_view.Substring(2, 4), StringValueIs("€€")); - EXPECT_THAT(unicode_cord.Substring(2), StringValueIs("€€€€")); - EXPECT_THAT(unicode_view.Substring(2), StringValueIs("€€€€")); + EXPECT_THAT(unicode_cord.Substring(0, 2, arena()), StringValueIs("€€")); + EXPECT_THAT(unicode_view.Substring(0, 2, arena()), StringValueIs("€€")); + EXPECT_THAT(unicode_cord.Substring(1, 2, arena()), StringValueIs("€")); + EXPECT_THAT(unicode_view.Substring(1, 2, arena()), StringValueIs("€")); + EXPECT_THAT(unicode_cord.Substring(2, 4, arena()), StringValueIs("€€")); + EXPECT_THAT(unicode_view.Substring(2, 4, arena()), StringValueIs("€€")); + EXPECT_THAT(unicode_cord.Substring(2, arena()), StringValueIs("€€€€")); + EXPECT_THAT(unicode_view.Substring(2, arena()), StringValueIs("€€€€")); - EXPECT_THAT(unicode_cord.Substring(0, 7), + EXPECT_THAT(unicode_cord.Substring(0, 7, arena()), ErrorValueIs(absl::InvalidArgumentError( ".substring(, ): or is " "greater than .size()"))); - EXPECT_THAT(unicode_cord.Substring(-1), + EXPECT_THAT(unicode_cord.Substring(-1, arena()), ErrorValueIs(absl::InvalidArgumentError( ".substring(): is less than 0"))); } diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index 9cbdf83ac..d1e5b4882 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -82,7 +82,8 @@ absl::StatusOr> ProtoMessageCopyUsingSerialization( absl::StatusOr> ProtoMessageCopy( google::protobuf::Message* absl_nonnull to_message, const google::protobuf::Descriptor* absl_nonnull to_descriptor, - const google::protobuf::Message* absl_nonnull from_message) { + const google::protobuf::Message* absl_nonnull from_message, + google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(const auto* from_descriptor, GetDescriptor(*from_message)); if (to_descriptor == from_descriptor) { @@ -95,14 +96,14 @@ absl::StatusOr> ProtoMessageCopy( return ProtoMessageCopyUsingSerialization(to_message, from_message); } return TypeConversionError(from_descriptor->full_name(), - to_descriptor->full_name()); + to_descriptor->full_name(), arena); } absl::StatusOr> ProtoMessageFromValueImpl( const Value& value, const google::protobuf::DescriptorPool* absl_nonnull pool, google::protobuf::MessageFactory* absl_nonnull factory, well_known_types::Reflection* absl_nonnull well_known_types, - google::protobuf::Message* absl_nonnull message) { + google::protobuf::Message* absl_nonnull message, google::protobuf::Arena* absl_nonnull arena) { CEL_ASSIGN_OR_RETURN(const auto* to_desc, GetDescriptor(*message)); switch (to_desc->well_known_type()) { case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: { @@ -113,7 +114,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( message, static_cast(double_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: { if (auto double_value = value.AsDouble(); double_value) { @@ -123,13 +125,15 @@ absl::StatusOr> ProtoMessageFromValueImpl( double_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("int64 to int32 overflow"), arena); } CEL_RETURN_IF_ERROR(well_known_types->Int32Value().Initialize( message->GetDescriptor())); @@ -137,7 +141,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( message, static_cast(int_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: { if (auto int_value = value.AsInt(); int_value) { @@ -147,12 +152,14 @@ absl::StatusOr> ProtoMessageFromValueImpl( int_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena); } CEL_RETURN_IF_ERROR(well_known_types->UInt32Value().Initialize( message->GetDescriptor())); @@ -160,7 +167,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( message, static_cast(uint_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: { if (auto uint_value = value.AsUint(); uint_value) { @@ -170,7 +178,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( uint_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: { if (auto string_value = value.AsString(); string_value) { @@ -180,7 +189,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( string_value->NativeCord()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: { if (auto bytes_value = value.AsBytes(); bytes_value) { @@ -190,7 +200,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( bytes_value->NativeCord()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_BOOLVALUE: { if (auto bool_value = value.AsBool(); bool_value) { @@ -200,7 +211,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( bool_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_ANY: { google::protobuf::io::CordOutputStream serialized; @@ -259,7 +271,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( message, duration_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: { if (auto timestamp_value = value.AsTimestamp(); timestamp_value) { @@ -269,7 +282,8 @@ absl::StatusOr> ProtoMessageFromValueImpl( message, timestamp_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), to_desc->full_name()); + return TypeConversionError(value.GetTypeName(), to_desc->full_name(), + arena); } case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: { CEL_RETURN_IF_ERROR(value.ConvertToJson(pool, factory, message)); @@ -293,85 +307,95 @@ absl::StatusOr> ProtoMessageFromValueImpl( if (auto legacy_value = common_internal::AsLegacyStructValue(value); legacy_value) { const auto* from_message = legacy_value->message_ptr(); - return ProtoMessageCopy(message, to_desc, from_message); + return ProtoMessageCopy(message, to_desc, from_message, arena); } // Deal with modern values. if (auto parsed_message_value = value.AsParsedMessage(); parsed_message_value) { return ProtoMessageCopy(message, to_desc, - cel::to_address(*parsed_message_value)); + cel::to_address(*parsed_message_value), arena); } - return TypeConversionError(value.GetTypeName(), message->GetTypeName()); + return TypeConversionError(value.GetTypeName(), message->GetTypeName(), + arena); } // Converts a value to a specific protocol buffer map key. using ProtoMapKeyFromValueConverter = absl::StatusOr> (*)(const Value&, google::protobuf::MapKey&, - std::string&); + std::string&, + google::protobuf::Arena* absl_nonnull); absl::StatusOr> ProtoBoolMapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto bool_value = value.AsBool(); bool_value) { key.SetBoolValue(bool_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bool"); + return TypeConversionError(value.GetTypeName(), "bool", arena); } absl::StatusOr> ProtoInt32MapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From(absl::OutOfRangeError("int64 to int32 overflow"), + arena); } key.SetInt32Value(static_cast(int_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> ProtoInt64MapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { key.SetInt64Value(int_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> ProtoUInt32MapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena); } key.SetUInt32Value(static_cast(uint_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> ProtoUInt64MapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string&) { + const Value& value, google::protobuf::MapKey& key, std::string&, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { key.SetUInt64Value(uint_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> ProtoStringMapKeyFromValueConverter( - const Value& value, google::protobuf::MapKey& key, std::string& key_string) { + const Value& value, google::protobuf::MapKey& key, std::string& key_string, + google::protobuf::Arena* absl_nonnull arena) { if (auto string_value = value.AsString(); string_value) { key_string = string_value->NativeString(); key.SetStringValue(key_string); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "string"); + return TypeConversionError(value.GetTypeName(), "string", arena); } // Gets the converter for converting from values to protocol buffer map key. @@ -403,49 +427,51 @@ using ProtoMapValueFromValueConverter = const Value&, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef&); + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef&, + google::protobuf::Arena* absl_nonnull); absl::StatusOr> ProtoBoolMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto bool_value = value.AsBool(); bool_value) { value_ref.SetBoolValue(bool_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bool"); + return TypeConversionError(value.GetTypeName(), "bool", arena); } absl::StatusOr> ProtoInt32MapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From(absl::OutOfRangeError("int64 to int32 overflow"), + arena); } value_ref.SetInt32Value(static_cast(int_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> ProtoInt64MapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { value_ref.SetInt64Value(int_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> @@ -453,16 +479,17 @@ ProtoUInt32MapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena); } value_ref.SetUInt32Value(static_cast(uint_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> @@ -470,26 +497,26 @@ ProtoUInt64MapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { value_ref.SetUInt64Value(uint_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> ProtoFloatMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto double_value = value.AsDouble(); double_value) { value_ref.SetFloatValue(double_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena); } absl::StatusOr> @@ -497,26 +524,26 @@ ProtoDoubleMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto double_value = value.AsDouble(); double_value) { value_ref.SetDoubleValue(double_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena); } absl::StatusOr> ProtoBytesMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto bytes_value = value.AsBytes(); bytes_value) { value_ref.SetStringValue(bytes_value->NativeString()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bytes"); + return TypeConversionError(value.GetTypeName(), "bytes", arena); } absl::StatusOr> @@ -524,43 +551,45 @@ ProtoStringMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto string_value = value.AsString(); string_value) { value_ref.SetStringValue(string_value->NativeString()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "string"); + return TypeConversionError(value.GetTypeName(), "string", arena); } absl::StatusOr> ProtoNullMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (value.IsNull() || value.IsInt()) { value_ref.SetEnumValue(0); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "google.protobuf.NullValue"); + return TypeConversionError(value.GetTypeName(), "google.protobuf.NullValue", + arena); } absl::StatusOr> ProtoEnumMapValueFromValueConverter( const Value& value, const google::protobuf::FieldDescriptor* absl_nonnull field, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, - well_known_types::Reflection* absl_nonnull, - google::protobuf::MapValueRef& value_ref) { + well_known_types::Reflection* absl_nonnull, google::protobuf::MapValueRef& value_ref, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From(absl::OutOfRangeError("int64 to int32 overflow"), + arena); } value_ref.SetEnumValue(static_cast(int_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "enum"); + return TypeConversionError(value.GetTypeName(), "enum", arena); } absl::StatusOr> @@ -569,9 +598,9 @@ ProtoMessageMapValueFromValueConverter( const google::protobuf::DescriptorPool* absl_nonnull pool, google::protobuf::MessageFactory* absl_nonnull factory, well_known_types::Reflection* absl_nonnull well_known_types, - google::protobuf::MapValueRef& value_ref) { + google::protobuf::MapValueRef& value_ref, google::protobuf::Arena* absl_nonnull arena) { return ProtoMessageFromValueImpl(value, pool, factory, well_known_types, - value_ref.MutableMessageValue()); + value_ref.MutableMessageValue(), arena); } // Gets the converter for converting from values to protocol buffer map value. @@ -621,7 +650,8 @@ using ProtoRepeatedFieldFromValueMutator = google::protobuf::MessageFactory* absl_nonnull, well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull, google::protobuf::Message* absl_nonnull, - const google::protobuf::FieldDescriptor* absl_nonnull, const Value&); + const google::protobuf::FieldDescriptor* absl_nonnull, const Value&, + google::protobuf::Arena* absl_nonnull); absl::StatusOr> ProtoBoolRepeatedFieldFromValueMutator( @@ -630,12 +660,13 @@ ProtoBoolRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto bool_value = value.AsBool(); bool_value) { reflection->AddBool(message, field, bool_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bool"); + return TypeConversionError(value.GetTypeName(), "bool", arena); } absl::StatusOr> @@ -645,17 +676,19 @@ ProtoInt32RepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From(absl::OutOfRangeError("int64 to int32 overflow"), + arena); } reflection->AddInt32(message, field, static_cast(int_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> @@ -665,12 +698,13 @@ ProtoInt64RepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto int_value = value.AsInt(); int_value) { reflection->AddInt64(message, field, int_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena); } absl::StatusOr> @@ -680,16 +714,18 @@ ProtoUInt32RepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena); } reflection->AddUInt32(message, field, static_cast(uint_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> @@ -699,12 +735,13 @@ ProtoUInt64RepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto uint_value = value.AsUint(); uint_value) { reflection->AddUInt64(message, field, uint_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena); } absl::StatusOr> @@ -714,13 +751,14 @@ ProtoFloatRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto double_value = value.AsDouble(); double_value) { reflection->AddFloat(message, field, static_cast(double_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena); } absl::StatusOr> @@ -730,12 +768,13 @@ ProtoDoubleRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto double_value = value.AsDouble(); double_value) { reflection->AddDouble(message, field, double_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena); } absl::StatusOr> @@ -745,12 +784,13 @@ ProtoBytesRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto bytes_value = value.AsBytes(); bytes_value) { reflection->AddString(message, field, bytes_value->NativeString()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bytes"); + return TypeConversionError(value.GetTypeName(), "bytes", arena); } absl::StatusOr> @@ -760,12 +800,13 @@ ProtoStringRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (auto string_value = value.AsString(); string_value) { reflection->AddString(message, field, string_value->NativeString()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "string"); + return TypeConversionError(value.GetTypeName(), "string", arena); } absl::StatusOr> @@ -775,12 +816,13 @@ ProtoNullRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { if (value.IsNull() || value.IsInt()) { reflection->AddEnumValue(message, field, 0); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "null_type"); + return TypeConversionError(value.GetTypeName(), "null_type", arena); } absl::StatusOr> @@ -790,19 +832,21 @@ ProtoEnumRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { const auto* enum_descriptor = field->enum_type(); if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { return TypeConversionError(value.GetTypeName(), - enum_descriptor->full_name()); + enum_descriptor->full_name(), arena); } reflection->AddEnumValue(message, field, static_cast(int_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), enum_descriptor->full_name()); + return TypeConversionError(value.GetTypeName(), enum_descriptor->full_name(), + arena); } absl::StatusOr> @@ -812,7 +856,8 @@ ProtoMessageRepeatedFieldFromValueMutator( well_known_types::Reflection* absl_nonnull well_known_types, const google::protobuf::Reflection* absl_nonnull reflection, google::protobuf::Message* absl_nonnull message, - const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value) { + const google::protobuf::FieldDescriptor* absl_nonnull field, const Value& value, + google::protobuf::Arena* absl_nonnull arena) { // If the value is null and the target repeated field is anything except // google.protobuf.{Any,ListValue,Struct,Value}, it should be pruned. if (value.IsNull()) { @@ -826,7 +871,7 @@ ProtoMessageRepeatedFieldFromValueMutator( } auto* element = reflection->AddMessage(message, field, factory); auto result = ProtoMessageFromValueImpl(value, pool, factory, - well_known_types, element); + well_known_types, element, arena); if (!result.ok() || result->has_value()) { reflection->RemoveLast(message, field); } @@ -898,7 +943,7 @@ class MessageValueBuilderImpl { if (field == nullptr) { field = descriptor_pool_->FindExtensionByPrintableName(descriptor_, name); if (field == nullptr) { - return NoSuchFieldError(name); + return NoSuchFieldError(name, arena_); } } return SetField(field, std::move(value)); @@ -908,12 +953,12 @@ class MessageValueBuilderImpl { Value value) { if (number < std::numeric_limits::min() || number > std::numeric_limits::max()) { - return NoSuchFieldError(absl::StrCat(number)); + return NoSuchFieldError(absl::StrCat(number), arena_); } const auto* field = descriptor_->FindFieldByNumber(static_cast(number)); if (field == nullptr) { - return NoSuchFieldError(absl::StrCat(number)); + return NoSuchFieldError(absl::StrCat(number), arena_); } return SetField(field, std::move(value)); } @@ -932,7 +977,7 @@ class MessageValueBuilderImpl { const google::protobuf::FieldDescriptor* absl_nonnull field, Value value) { auto map_value = value.AsMap(); if (!map_value) { - return TypeConversionError(value.GetTypeName(), "map"); + return TypeConversionError(value.GetTypeName(), "map", arena_); } CEL_ASSIGN_OR_RETURN(auto key_converter, GetProtoMapKeyFromValueConverter( @@ -953,7 +998,7 @@ class MessageValueBuilderImpl { google::protobuf::MapKey proto_key; CEL_ASSIGN_OR_RETURN( error_value, - (*key_converter)(entry_key, proto_key, proto_key_string)); + (*key_converter)(entry_key, proto_key, proto_key_string, arena_)); if (error_value) { return false; } @@ -977,7 +1022,7 @@ class MessageValueBuilderImpl { error_value, (*value_converter)(entry_value, map_value_field, descriptor_pool_, message_factory_, &well_known_types_, - proto_value)); + proto_value, arena_)); if (error_value) { return false; } @@ -994,7 +1039,8 @@ class MessageValueBuilderImpl { const google::protobuf::FieldDescriptor* absl_nonnull field, Value value) { auto list_value = value.AsList(); if (!list_value) { - return TypeConversionError(value.GetTypeName(), "list").NativeValue(); + return TypeConversionError(value.GetTypeName(), "list", arena_) + .NativeValue(); } CEL_ASSIGN_OR_RETURN(auto accessor, GetProtoRepeatedFieldFromValueMutator(field)); @@ -1016,7 +1062,7 @@ class MessageValueBuilderImpl { CEL_ASSIGN_OR_RETURN(error_value, (*accessor)(descriptor_pool_, message_factory_, &well_known_types_, reflection_, - message_, field, element)); + message_, field, element, arena_)); return !error_value; }, descriptor_pool_, message_factory_, arena_)); @@ -1031,61 +1077,62 @@ class MessageValueBuilderImpl { reflection_->SetBool(message_, field, bool_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bool"); + return TypeConversionError(value.GetTypeName(), "bool", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() < std::numeric_limits::min() || int_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue(absl::OutOfRangeError("int64 to int32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("int64 to int32 overflow"), arena_); } reflection_->SetInt32(message_, field, static_cast(int_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { if (auto int_value = value.AsInt(); int_value) { reflection_->SetInt64(message_, field, int_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "int"); + return TypeConversionError(value.GetTypeName(), "int", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { if (auto uint_value = value.AsUint(); uint_value) { if (uint_value->NativeValue() > std::numeric_limits::max()) { - return ErrorValue( - absl::OutOfRangeError("uint64 to uint32 overflow")); + return ErrorValue::From( + absl::OutOfRangeError("uint64 to uint32 overflow"), arena_); } reflection_->SetUInt32( message_, field, static_cast(uint_value->NativeValue())); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { if (auto uint_value = value.AsUint(); uint_value) { reflection_->SetUInt64(message_, field, uint_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "uint"); + return TypeConversionError(value.GetTypeName(), "uint", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { if (auto double_value = value.AsDouble(); double_value) { reflection_->SetFloat(message_, field, double_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { if (auto double_value = value.AsDouble(); double_value) { reflection_->SetDouble(message_, field, double_value->NativeValue()); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "double"); + return TypeConversionError(value.GetTypeName(), "double", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { if (field->type() == google::protobuf::FieldDescriptor::TYPE_BYTES) { @@ -1099,7 +1146,7 @@ class MessageValueBuilderImpl { })); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "bytes"); + return TypeConversionError(value.GetTypeName(), "bytes", arena_); } if (auto string_value = value.AsString(); string_value) { string_value->NativeValue(absl::Overload( @@ -1111,7 +1158,7 @@ class MessageValueBuilderImpl { })); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "string"); + return TypeConversionError(value.GetTypeName(), "string", arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { if (field->enum_type()->full_name() == "google.protobuf.NullValue") { @@ -1119,7 +1166,7 @@ class MessageValueBuilderImpl { reflection_->SetEnumValue(message_, field, 0); return std::nullopt; } - return TypeConversionError(value.GetTypeName(), "null_type"); + return TypeConversionError(value.GetTypeName(), "null_type", arena_); } if (auto int_value = value.AsInt(); int_value) { if (int_value->NativeValue() >= std::numeric_limits::min() && @@ -1130,7 +1177,7 @@ class MessageValueBuilderImpl { } } return TypeConversionError(value.GetTypeName(), - field->enum_type()->full_name()); + field->enum_type()->full_name(), arena_); } case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { switch (field->message_type()->well_known_type()) { @@ -1149,7 +1196,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_INT32VALUE: { if (value.IsNull()) { @@ -1172,7 +1220,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_INT64VALUE: { if (value.IsNull()) { @@ -1189,7 +1238,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT32VALUE: { if (value.IsNull()) { @@ -1210,7 +1260,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_UINT64VALUE: { if (value.IsNull()) { @@ -1227,7 +1278,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_FLOATVALUE: { if (value.IsNull()) { @@ -1244,7 +1296,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_DOUBLEVALUE: { if (value.IsNull()) { @@ -1261,7 +1314,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_BYTESVALUE: { if (value.IsNull()) { @@ -1278,7 +1332,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_STRINGVALUE: { if (value.IsNull()) { @@ -1295,7 +1350,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_DURATION: { if (value.IsNull()) { @@ -1313,7 +1369,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_TIMESTAMP: { if (value.IsNull()) { @@ -1330,7 +1387,8 @@ class MessageValueBuilderImpl { return std::nullopt; } return TypeConversionError(value.GetTypeName(), - field->message_type()->full_name()); + field->message_type()->full_name(), + arena_); } case google::protobuf::Descriptor::WELLKNOWNTYPE_VALUE: { CEL_RETURN_IF_ERROR( @@ -1416,7 +1474,8 @@ class MessageValueBuilderImpl { } return ProtoMessageFromValueImpl( value, descriptor_pool_, message_factory_, &well_known_types_, - reflection_->MutableMessage(message_, field, message_factory_)); + reflection_->MutableMessage(message_, field, message_factory_), + arena_); } default: return absl::InternalError( diff --git a/common/values/value_builder.cc b/common/values/value_builder.cc index 825fafeaf..78d7a57d8 100644 --- a/common/values/value_builder.cc +++ b/common/values/value_builder.cc @@ -309,7 +309,7 @@ class CompatListValueImpl final : public CompatListValue { } if (ABSL_PREDICT_FALSE(index < 0 || index >= size())) { return CelValue::CreateError(google::protobuf::Arena::Create( - arena, IndexOutOfBoundsError(index).ToStatus())); + arena, common_internal::MakeIndexOutOfBoundsError(index))); } return common_internal::UnsafeLegacyValue( elements_[index], @@ -326,7 +326,7 @@ class CompatListValueImpl final : public CompatListValue { google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const override { if (index >= elements_.size()) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); } else { *result = elements_[index]; } @@ -446,7 +446,7 @@ class MutableCompatListValueImpl final : public MutableCompatListValue { } if (ABSL_PREDICT_FALSE(index < 0 || index >= size())) { return CelValue::CreateError(google::protobuf::Arena::Create( - arena, IndexOutOfBoundsError(index).ToStatus())); + arena, common_internal::MakeIndexOutOfBoundsError(index))); } return common_internal::UnsafeLegacyValue( elements_[index], /*stable=*/false, @@ -478,7 +478,7 @@ class MutableCompatListValueImpl final : public MutableCompatListValue { google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const override { if (index >= elements_.size()) { - *result = IndexOutOfBoundsError(index); + *result = IndexOutOfBoundsError(index, arena); } else { *result = elements_[index]; } @@ -797,8 +797,8 @@ absl::StatusOr ValueToJsonString(const Value& value) { case ValueKind::kString: return value.GetString().NativeString(); default: - return TypeConversionError(value.GetRuntimeType(), StringType()) - .ToStatus(); + return common_internal::MakeTypeConversionError(value.GetRuntimeType(), + StringType()); } } diff --git a/common/values/value_variant.cc b/common/values/value_variant.cc index 7c9981d83..5f240f79b 100644 --- a/common/values/value_variant.cc +++ b/common/values/value_variant.cc @@ -21,7 +21,6 @@ #include "absl/base/optimization.h" #include "absl/log/absl_check.h" -#include "common/values/error_value.h" #include "common/values/unknown_value.h" #include "common/values/values.h" @@ -31,9 +30,6 @@ void ValueVariant::SlowCopyConstruct(const ValueVariant& other) noexcept { ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); switch (index_) { - case ValueIndex::kError: - ::new (static_cast(&raw_[0])) ErrorValue(*other.At()); - break; case ValueIndex::kUnknown: ::new (static_cast(&raw_[0])) UnknownValue(*other.At()); @@ -47,10 +43,6 @@ void ValueVariant::SlowMoveConstruct(ValueVariant& other) noexcept { ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); switch (index_) { - case ValueIndex::kError: - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - break; case ValueIndex::kUnknown: ::new (static_cast(&raw_[0])) UnknownValue(std::move(*other.At())); @@ -64,9 +56,6 @@ void ValueVariant::SlowDestruct() noexcept { ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); switch (index_) { - case ValueIndex::kError: - At()->~ErrorValue(); - break; case ValueIndex::kUnknown: At()->~UnknownValue(); break; @@ -81,10 +70,6 @@ void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, if (trivial) { switch (other.index_) { - case ValueIndex::kError: - ::new (static_cast(&raw_[0])) - ErrorValue(*other.At()); - break; case ValueIndex::kUnknown: ::new (static_cast(&raw_[0])) UnknownValue(*other.At()); @@ -97,9 +82,6 @@ void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, flags_ = other.flags_; } else if (other_trivial) { switch (index_) { - case ValueIndex::kError: - At()->~ErrorValue(); - break; case ValueIndex::kUnknown: At()->~UnknownValue(); break; @@ -109,31 +91,8 @@ void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, FastCopyAssign(other); } else { switch (index_) { - case ValueIndex::kError: - switch (other.index_) { - case ValueIndex::kError: - *At() = *other.At(); - break; - case ValueIndex::kUnknown: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; case ValueIndex::kUnknown: switch (other.index_) { - case ValueIndex::kError: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; case ValueIndex::kUnknown: At()->~UnknownValue(); ::new (static_cast(&raw_[0])) @@ -158,10 +117,6 @@ void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, if (trivial) { switch (other.index_) { - case ValueIndex::kError: - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - break; case ValueIndex::kUnknown: ::new (static_cast(&raw_[0])) UnknownValue(std::move(*other.At())); @@ -174,9 +129,6 @@ void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, flags_ = other.flags_; } else if (other_trivial) { switch (index_) { - case ValueIndex::kError: - At()->~ErrorValue(); - break; case ValueIndex::kUnknown: At()->~UnknownValue(); break; @@ -186,31 +138,8 @@ void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, FastMoveAssign(other); } else { switch (index_) { - case ValueIndex::kError: - switch (other.index_) { - case ValueIndex::kError: - *At() = std::move(*other.At()); - break; - case ValueIndex::kUnknown: - At()->~ErrorValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; case ValueIndex::kUnknown: switch (other.index_) { - case ValueIndex::kError: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - ErrorValue(std::move(*other.At())); - index_ = other.index_; - kind_ = other.kind_; - break; case ValueIndex::kUnknown: *At() = std::move(*other.At()); break; @@ -236,11 +165,6 @@ void ValueVariant::SlowSwap(ValueVariant& lhs, ValueVariant& rhs, // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) std::memcpy(tmp, std::addressof(lhs), sizeof(ValueVariant)); switch (rhs.index_) { - case ValueIndex::kError: - ::new (static_cast(&lhs.raw_[0])) - ErrorValue(*rhs.At()); - rhs.At()->~ErrorValue(); - break; case ValueIndex::kUnknown: ::new (static_cast(&lhs.raw_[0])) UnknownValue(*rhs.At()); @@ -261,11 +185,6 @@ void ValueVariant::SlowSwap(ValueVariant& lhs, ValueVariant& rhs, // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) std::memcpy(tmp, std::addressof(rhs), sizeof(ValueVariant)); switch (lhs.index_) { - case ValueIndex::kError: - ::new (static_cast(&rhs.raw_[0])) - ErrorValue(*lhs.At()); - lhs.At()->~ErrorValue(); - break; case ValueIndex::kUnknown: ::new (static_cast(&rhs.raw_[0])) UnknownValue(*lhs.At()); diff --git a/common/values/value_variant.h b/common/values/value_variant.h index 5a26a742f..e7c199834 100644 --- a/common/values/value_variant.h +++ b/common/values/value_variant.h @@ -88,8 +88,8 @@ enum class ValueIndex : uint8_t { kOpaque, kBytes, kString, - // Keep non-trivial alternatives together to aid in compiling optimizations. kError, + // Keep non-trivial alternatives together to aid in compiling optimizations. kUnknown, }; @@ -351,7 +351,7 @@ template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kBytes; static constexpr ValueKind kKind = BytesValue::kKind; - static constexpr bool kAlwaysTrivial = false; + static constexpr bool kAlwaysTrivial = true; static ValueFlags Flags(const BytesValue* absl_nonnull alternative) { return ValueFlags::kNone; @@ -362,7 +362,7 @@ template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kString; static constexpr ValueKind kKind = StringValue::kKind; - static constexpr bool kAlwaysTrivial = false; + static constexpr bool kAlwaysTrivial = true; static ValueFlags Flags(const StringValue* absl_nonnull alternative) { return ValueFlags::kNone; @@ -373,12 +373,10 @@ template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kError; static constexpr ValueKind kKind = ErrorValue::kKind; - static constexpr bool kAlwaysTrivial = false; + static constexpr bool kAlwaysTrivial = true; static ValueFlags Flags(const ErrorValue* absl_nonnull alternative) { - return ArenaTraits::trivially_destructible(*alternative) - ? ValueFlags::kNone - : ValueFlags::kNonTrivial; + return ValueFlags::kNone; } }; diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 26b581ef4..d108aa8de 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -926,6 +926,7 @@ cc_library( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", + "@com_google_protobuf//:protobuf", ], ) diff --git a/eval/eval/attribute_utility.cc b/eval/eval/attribute_utility.cc index 274deb78c..af63e9f91 100644 --- a/eval/eval/attribute_utility.cc +++ b/eval/eval/attribute_utility.cc @@ -20,6 +20,7 @@ #include "eval/internal/errors.h" #include "internal/status_macros.h" #include "runtime/internal/attribute_matcher.h" +#include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -209,10 +210,10 @@ UnknownValue AttributeUtility::CreateUnknownSet(cel::Attribute attr) const { } absl::StatusOr AttributeUtility::CreateMissingAttributeError( - const cel::Attribute& attr) const { + const cel::Attribute& attr, google::protobuf::Arena* arena) const { CEL_ASSIGN_OR_RETURN(std::string message, attr.AsString()); - return cel::ErrorValue( - cel::runtime_internal::CreateMissingAttributeError(message)); + return cel::ErrorValue::From( + cel::runtime_internal::CreateMissingAttributeError(message), arena); } UnknownValue AttributeUtility::CreateUnknownSet( diff --git a/eval/eval/attribute_utility.h b/eval/eval/attribute_utility.h index 94a5158f0..51ffc5d47 100644 --- a/eval/eval/attribute_utility.h +++ b/eval/eval/attribute_utility.h @@ -14,6 +14,7 @@ #include "common/value.h" #include "eval/eval/attribute_trail.h" #include "runtime/internal/attribute_matcher.h" +#include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -149,7 +150,7 @@ class AttributeUtility { // Factory function for missing attribute errors. absl::StatusOr CreateMissingAttributeError( - const cel::Attribute& attr) const; + const cel::Attribute& attr, google::protobuf::Arena* arena) const; // Create an initial UnknownSet from a single missing function call. cel::UnknownValue CreateUnknownSet( diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index 65636f347..9e8344fcf 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -108,7 +108,8 @@ absl::Status CreateListStep::DoEvaluate(ExecutionFrame* frame, } CEL_RETURN_IF_ERROR(builder->Add(std::move(optional_arg_value))); } else { - *result = cel::TypeConversionError(arg.GetTypeName(), "optional_type"); + *result = cel::TypeConversionError(arg.GetTypeName(), "optional_type", + frame->arena()); return absl::OkStatus(); } } else { @@ -162,7 +163,7 @@ class CreateListDirectStep : public DirectExpressionStep { if (frame.attribute_utility().CheckForMissingAttribute(tmp_attr)) { CEL_ASSIGN_OR_RETURN( result, frame.attribute_utility().CreateMissingAttributeError( - tmp_attr.attribute())); + tmp_attr.attribute(), frame.arena())); return absl::OkStatus(); } } @@ -200,8 +201,8 @@ class CreateListDirectStep : public DirectExpressionStep { CEL_RETURN_IF_ERROR(builder->Add(std::move(optional_arg_value))); continue; } - result = - cel::TypeConversionError(result.GetTypeName(), "optional_type"); + result = cel::TypeConversionError(result.GetTypeName(), "optional_type", + frame.arena()); return absl::OkStatus(); } diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index 2faf1f11e..a495aad11 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -307,9 +307,9 @@ TEST(CreateDirectListStep, ForwardFirstError) { std::vector> deps; deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError("test1")), -1)); + cel::ErrorValue::From(absl::InternalError("test1"), &arena), -1)); deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError("test2")), -1)); + cel::ErrorValue::From(absl::InternalError("test2"), &arena), -1)); auto step = CreateDirectListStep(std::move(deps), {}, -1); cel::Value result; @@ -387,9 +387,9 @@ TEST(CreateDirectListStep, ErrorBeforeUnknown) { std::vector> deps; deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError("test1")), -1)); + cel::ErrorValue::From(absl::InternalError("test1"), &arena), -1)); deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError("test2")), -1)); + cel::ErrorValue::From(absl::InternalError("test2"), &arena), -1)); auto step = CreateDirectListStep(std::move(deps), {}, -1); cel::Value result; diff --git a/eval/eval/create_map_step.cc b/eval/eval/create_map_step.cc index 4b27e5e30..f8f099047 100644 --- a/eval/eval/create_map_step.cc +++ b/eval/eval/create_map_step.cc @@ -90,7 +90,8 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( for (size_t i = 0; i < entry_count_; i += 1) { const auto& map_key = args[2 * i]; - CEL_RETURN_IF_ERROR(cel::CheckMapKey(map_key)).With(ErrorValueReturn()); + CEL_RETURN_IF_ERROR(cel::CheckMapKey(map_key)) + .With(ErrorValueReturn(frame->arena())); const auto& map_value = args[(2 * i) + 1]; if (optional_indices_.contains(static_cast(i))) { if (auto optional_map_value = map_value.AsOptional(); @@ -108,7 +109,7 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( builder->Put(map_key, std::move(optional_map_value_value))); } else { return cel::TypeConversionError(map_value.DebugString(), - "optional_type"); + "optional_type", frame->arena()); } } else { CEL_RETURN_IF_ERROR(builder->Put(map_key, map_value)); @@ -182,7 +183,8 @@ absl::Status DirectCreateMapStep::Evaluate( } } - CEL_RETURN_IF_ERROR(cel::CheckMapKey(key)).With(ErrorValueAssign(result)); + CEL_RETURN_IF_ERROR(cel::CheckMapKey(key)) + .With(ErrorValueAssign(result, frame.arena())); CEL_RETURN_IF_ERROR( deps_[map_value_index]->Evaluate(frame, value, tmp_attr)); @@ -222,7 +224,8 @@ absl::Status DirectCreateMapStep::Evaluate( builder->Put(std::move(key), std::move(optional_map_value_value))); continue; } - result = cel::TypeConversionError(value.DebugString(), "optional_type"); + result = cel::TypeConversionError(value.DebugString(), "optional_type", + frame.arena()); return absl::OkStatus(); } diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index bce2a3ea9..3ac6caabe 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -117,7 +117,8 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( return std::move(*error_value); } } else { - return cel::TypeConversionError(arg.DebugString(), "optional_type"); + return cel::TypeConversionError(arg.DebugString(), "optional_type", + frame->arena()); } } else { CEL_ASSIGN_OR_RETURN(absl::optional error_value, @@ -230,7 +231,7 @@ absl::Status DirectCreateStructStep::Evaluate(ExecutionFrameBase& frame, continue; } else { result = cel::TypeConversionError(field_value.DebugString(), - "optional_type"); + "optional_type", frame.arena()); return absl::OkStatus(); } } diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index 76031e169..f479d5d3b 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -228,7 +228,7 @@ Value MakeValue(InputType type, google::protobuf::Arena* absl_nonnull arena) { } case InputType::kError: default: - return ErrorValue(absl::InternalError("error")); + return ErrorValue::From(absl::InternalError("error"), arena); } } diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index e3985d183..23d1f5304 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -46,7 +46,7 @@ absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, frame.attribute_utility().CheckForMissingAttribute(attribute)) { CEL_ASSIGN_OR_RETURN( result, frame.attribute_utility().CreateMissingAttributeError( - attribute.attribute())); + attribute.attribute(), frame.arena())); return absl::OkStatus(); } if (frame.unknown_processing_enabled() && diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index a7528687b..bd6803059 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -385,7 +385,8 @@ void EvaluateBoolLogicStep(BoolLogicKind kind, size_t num_args, cel::Value result = args[error_pos.value()]; if (!result.IsError()) { - result = cel::ErrorValue(CreateNoMatchingOverloadError(op_name)); + result = cel::ErrorValue::From(CreateNoMatchingOverloadError(op_name), + frame.arena()); } frame.value_stack().PopAndPush(num_args, std::move(result)); } diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index 93f2fb888..8bd4834f9 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -356,7 +356,8 @@ UnknownValue MakeUnknownValue(std::string attr) { } std::unique_ptr MakeArgStep(OpArg arg, - absl::string_view name) { + absl::string_view name, + google::protobuf::Arena* arena) { switch (arg) { case OpArg::kTrue: return CreateConstValueDirectStep(BoolValue(true)); @@ -366,7 +367,7 @@ std::unique_ptr MakeArgStep(OpArg arg, return CreateConstValueDirectStep(MakeUnknownValue(std::string(name))); case OpArg::kError: return CreateConstValueDirectStep( - cel::ErrorValue(absl::InternalError(name))); + cel::ErrorValue::From(absl::InternalError(name), arena)); case OpArg::kInt: return CreateConstValueDirectStep(IntValue(42)); } @@ -388,9 +389,9 @@ TEST_P(DirectBinaryLogicStepTest, TestCases) { const BinaryTestCase& test_case = GetTestCase(); std::unique_ptr lhs = - MakeArgStep(test_case.arg0, "lhs"); + MakeArgStep(test_case.arg0, "lhs", &arena_); std::unique_ptr rhs = - MakeArgStep(test_case.arg1, "rhs"); + MakeArgStep(test_case.arg1, "rhs", &arena_); std::unique_ptr op = (test_case.op == BinaryOp::kAnd) @@ -579,7 +580,8 @@ class DirectUnaryLogicStepTest : public testing::TestWithParam { TEST_P(DirectUnaryLogicStepTest, TestCases) { const UnaryTestCase& test_case = GetTestCase(); - std::unique_ptr arg = MakeArgStep(test_case.arg, "arg"); + std::unique_ptr arg = + MakeArgStep(test_case.arg, "arg", &arena_); std::unique_ptr op = (test_case.op == UnaryOp::kNot) @@ -662,12 +664,13 @@ TEST(UnaryLogicStepTest, BooleanNot) { } TEST(UnaryLogicStepTest, NotStrictlyFalse) { + google::protobuf::Arena arena; + ExecutionPath path; path.push_back(ExpressionStep::MakeConstant( - cel::ErrorValue(absl::InternalError("error")))); + cel::ErrorValue::From(absl::InternalError("error"), &arena))); path.push_back(ExpressionStep::MakeNotStrictlyFalseStep()); - google::protobuf::Arena arena; cel::runtime_internal::RuntimeTypeProvider type_provider( cel::internal::GetTestingDescriptorPool()); FlatExpressionEvaluatorState state( diff --git a/eval/eval/optional_or_step_test.cc b/eval/eval/optional_or_step_test.cc index 14f1c3bd9..2962ee38c 100644 --- a/eval/eval/optional_or_step_test.cc +++ b/eval/eval/optional_or_step_test.cc @@ -76,7 +76,8 @@ std::unique_ptr MockExpectCallDirectStep() { .Times(1) .WillRepeatedly( [](ExecutionFrameBase& frame, Value& result, AttributeTrail& attr) { - result = ErrorValue(absl::InternalError("expected to be unused")); + result = ErrorValue::From( + absl::InternalError("expected to be unused"), frame.arena()); return absl::OkStatus(); }); return absl::WrapUnique(mock); @@ -122,7 +123,8 @@ TEST_F(OptionalOrTest, OptionalOrLeftErrorShortcutsRight) { std::unique_ptr step = CreateDirectOptionalOrStep( /*expr_id=*/-1, - CreateConstValueDirectStep(ErrorValue(absl::InternalError("error"))), + CreateConstValueDirectStep( + ErrorValue::From(absl::InternalError("error"), &arena_)), MockNeverCalledDirectStep(), /*is_or_value=*/false, /*short_circuiting=*/true); @@ -142,7 +144,8 @@ TEST_F(OptionalOrTest, OptionalOrLeftErrorExhaustiveRight) { std::unique_ptr step = CreateDirectOptionalOrStep( /*expr_id=*/-1, - CreateConstValueDirectStep(ErrorValue(absl::InternalError("error"))), + CreateConstValueDirectStep( + ErrorValue::From(absl::InternalError("error"), &arena_)), MockExpectCallDirectStep(), /*is_or_value=*/false, /*short_circuiting=*/false); @@ -308,7 +311,8 @@ TEST_F(OptionalOrTest, OptionalOrValueLeftErrorShortcutsRight) { std::unique_ptr step = CreateDirectOptionalOrStep( /*expr_id=*/-1, - CreateConstValueDirectStep(ErrorValue(absl::InternalError("error"))), + CreateConstValueDirectStep( + ErrorValue::From(absl::InternalError("error"), &arena_)), MockNeverCalledDirectStep(), /*is_or_value=*/true, /*short_circuiting=*/true); diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 1d6c337d6..a8f270698 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -62,7 +62,7 @@ absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, if (frame.missing_attribute_errors_enabled() && frame.attribute_utility().CheckForMissingAttribute(trail)) { auto result = frame.attribute_utility().CreateMissingAttributeError( - trail.attribute()); + trail.attribute(), frame.arena()); if (result.ok()) { return std::move(result).value(); diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index 75b987463..874b5ea76 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -1583,8 +1583,8 @@ TEST_F(DirectSelectStepTest, ForwardErrorValue) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; auto step = CreateDirectSelectStep( - CreateConstValueDirectStep(cel::ErrorValue(absl::InternalError("test1")), - -1), + CreateConstValueDirectStep( + cel::ErrorValue::From(absl::InternalError("test1"), &arena_), -1), "single_int64", /*test_only=*/false, -1, /*enable_wrapper_type_null_unboxing=*/true); diff --git a/eval/eval/ternary_step_test.cc b/eval/eval/ternary_step_test.cc index 9b17b5356..db1df46f9 100644 --- a/eval/eval/ternary_step_test.cc +++ b/eval/eval/ternary_step_test.cc @@ -259,7 +259,8 @@ TEST_P(TernaryStepDirectTest, ForwardError) { cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena_); - cel::Value error_value = cel::ErrorValue(absl::InternalError("test error")); + cel::Value error_value = + cel::ErrorValue::From(absl::InternalError("test error"), &arena_); std::unique_ptr step = CreateDirectTernaryStep( CreateConstValueDirectStep(error_value, -1), diff --git a/extensions/comprehensions_v2_functions.cc b/extensions/comprehensions_v2_functions.cc index bf23780c0..b6414b5d4 100644 --- a/extensions/comprehensions_v2_functions.cc +++ b/extensions/comprehensions_v2_functions.cc @@ -45,7 +45,7 @@ absl::StatusOr MapInsertKeyValue( // Fast path, runtime has given us a mutable map. We can mutate it directly // and return it. CEL_RETURN_IF_ERROR(mutable_map_value->Put(key, value)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); return map; } // Slow path, we have to make a copy. @@ -63,8 +63,8 @@ absl::StatusOr MapInsertKeyValue( return true; }, descriptor_pool, message_factory, arena)) - .With(ErrorValueReturn()); - CEL_RETURN_IF_ERROR(builder->Put(key, value)).With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); + CEL_RETURN_IF_ERROR(builder->Put(key, value)).With(ErrorValueReturn(arena)); return std::move(*builder).Build(); } @@ -85,7 +85,7 @@ absl::StatusOr MapInsertMap( return true; }, descriptor_pool, message_factory, arena)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); return map; } // Slow path, we have to make a copy. @@ -103,7 +103,7 @@ absl::StatusOr MapInsertMap( return true; }, descriptor_pool, message_factory, arena)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); CEL_RETURN_IF_ERROR( value.ForEach( [&builder](const Value& key, @@ -112,7 +112,7 @@ absl::StatusOr MapInsertMap( return true; }, descriptor_pool, message_factory, arena)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); return std::move(*builder).Build(); } diff --git a/extensions/math_ext.cc b/extensions/math_ext.cc index a31b112e8..78c066f9f 100644 --- a/extensions/math_ext.cc +++ b/extensions/math_ext.cc @@ -159,7 +159,7 @@ absl::StatusOr MaxList( iterator->Next(descriptor_pool, message_factory, arena, &value)); absl::StatusOr current = ValueToNumber(value, kMathMax); if (!current.ok()) { - return ErrorValue{current.status()}; + return ErrorValue::From(current.status(), arena); } CelNumber min = *current; while (iterator->HasNext()) { @@ -167,7 +167,7 @@ absl::StatusOr MaxList( iterator->Next(descriptor_pool, message_factory, arena, &value)); absl::StatusOr other = ValueToNumber(value, kMathMax); if (!other.ok()) { - return ErrorValue{other.status()}; + return ErrorValue::From(other.status(), arena); } min = MaxNumber(min, *other); } diff --git a/extensions/protobuf/value.h b/extensions/protobuf/value.h index 4336b3d68..e965835a0 100644 --- a/extensions/protobuf/value.h +++ b/extensions/protobuf/value.h @@ -89,9 +89,8 @@ inline absl::Status ProtoMessageFromValue(const Value& value, return absl::OkStatus(); } } - return TypeConversionError(value.GetRuntimeType(), - MessageType(dest_descriptor)) - .NativeValue(); + return common_internal::MakeTypeConversionError(value.GetRuntimeType(), + MessageType(dest_descriptor)); } } // namespace cel::extensions diff --git a/extensions/regex_ext.cc b/extensions/regex_ext.cc index 63bebe7cd..065adc76d 100644 --- a/extensions/regex_ext.cc +++ b/extensions/regex_ext.cc @@ -65,7 +65,7 @@ Value Extract(int regex_max_program_size, const StringValue& target, absl::string_view regex_view = regex.ToStringView(®ex_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); const int group_count = re2.NumberOfCapturingGroups(); if (group_count > 1) { return ErrorValue::From( @@ -99,7 +99,7 @@ Value ExtractAll(int regex_max_program_size, const StringValue& target, absl::string_view regex_view = regex.ToStringView(®ex_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); const int group_count = re2.NumberOfCapturingGroups(); if (group_count > 1) { return ErrorValue::From( @@ -162,7 +162,7 @@ Value ReplaceAll(int regex_max_program_size, const StringValue& target, replacement.ToStringView(&replacement_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); std::string error_string; if (!re2.CheckRewriteString(replacement_view, &error_string)) { return ErrorValue::From( @@ -200,7 +200,7 @@ Value ReplaceN(int regex_max_program_size, const StringValue& target, replacement.ToStringView(&replacement_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); std::string error_string; if (!re2.CheckRewriteString(replacement_view, &error_string)) { return ErrorValue::From( diff --git a/extensions/regex_functions.cc b/extensions/regex_functions.cc index 249ecc563..3935772a2 100644 --- a/extensions/regex_functions.cc +++ b/extensions/regex_functions.cc @@ -65,7 +65,7 @@ Value ExtractString(int regex_max_program_size, const StringValue& target, RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); std::string output; bool result = RE2::Extract(target_view, re2, rewrite_view, &output); if (!result) { @@ -89,7 +89,7 @@ Value CaptureString(int regex_max_program_size, const StringValue& target, absl::string_view target_view = target.ToStringView(&target_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); std::string output; bool result = RE2::FullMatch(target_view, re2, &output); if (!result) { @@ -117,7 +117,7 @@ absl::StatusOr CaptureStringN( absl::string_view regex_view = regex.ToStringView(®ex_scratch); RE2 re2(regex_view, cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, regex_max_program_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(arena)); const int capturing_groups_count = re2.NumberOfCapturingGroups(); const auto& named_capturing_groups_map = re2.CapturingGroupNames(); if (capturing_groups_count <= 0) { diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 7bca6414c..5244907e6 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -696,7 +696,7 @@ absl::StatusOr> CheckForMarkedAttributes( if (frame.missing_attribute_errors_enabled() && frame.attribute_utility().CheckForMissingAttribute(attribute_trail)) { return frame.attribute_utility().CreateMissingAttributeError( - attribute_trail.attribute()); + attribute_trail.attribute(), frame.arena()); } return std::nullopt; diff --git a/extensions/select_optimization_test.cc b/extensions/select_optimization_test.cc index b1213142a..ecc78ae9e 100644 --- a/extensions/select_optimization_test.cc +++ b/extensions/select_optimization_test.cc @@ -266,7 +266,7 @@ class TestPartialQualifyStruct : public CustomStructValueInterface { *result = leaf_value_; return absl::OkStatus(); } - return NoSuchFieldError(name).ToStatus(); + return common_internal::MakeNoSuchFieldError(name); } absl::Status GetFieldByNumber(int64_t number, diff --git a/extensions/strings.cc b/extensions/strings.cc index ad3719113..831600353 100644 --- a/extensions/strings.cc +++ b/extensions/strings.cc @@ -108,8 +108,9 @@ absl::StatusOr Replace1( return string.Replace(old_sub, new_sub, -1, arena); } -Value CharAt(const StringValue& string, int64_t pos) { - return string.CharAt(pos); +Value CharAt(const StringValue& string, int64_t pos, + const Function::InvokeContext& context) { + return string.CharAt(pos, context.arena()); } int64_t IndexOf2(const StringValue& haystack, const StringValue& needle) { @@ -140,12 +141,14 @@ Value LastIndexOf3(const StringValue& haystack, const StringValue& needle, return IntValue(haystack.LastIndexOf(needle, pos).value_or(-1)); } -Value Substring2(const StringValue& string, int64_t start) { - return string.Substring(start); +Value Substring2(const StringValue& string, int64_t start, + const Function::InvokeContext& context) { + return string.Substring(start, context.arena()); } -Value Substring3(const StringValue& string, int64_t start, int64_t end) { - return string.Substring(start, end); +Value Substring3(const StringValue& string, int64_t start, int64_t end, + const Function::InvokeContext& context) { + return string.Substring(start, end, context.arena()); } StringValue Trim(const StringValue& string) { return string.Trim(); } diff --git a/runtime/function_adapter_test.cc b/runtime/function_adapter_test.cc index df5f50362..5895df87e 100644 --- a/runtime/function_adapter_test.cc +++ b/runtime/function_adapter_test.cc @@ -210,8 +210,9 @@ TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionAny) { TEST_F(FunctionAdapterTest, UnaryFunctionAdapterWrapFunctionReturnError) { using FunctionAdapter = UnaryFunctionAdapter; std::unique_ptr wrapped = - FunctionAdapter::WrapFunction([](uint64_t x) -> Value { - return ErrorValue(absl::InvalidArgumentError("test_error")); + FunctionAdapter::WrapFunction([this](uint64_t x) -> Value { + return ErrorValue::From(absl::InvalidArgumentError("test_error"), + arena()); }); std::vector args{UintValue(44)}; @@ -539,8 +540,9 @@ TEST_F(FunctionAdapterTest, BinaryFunctionAdapterWrapFunctionAny) { TEST_F(FunctionAdapterTest, BinaryFunctionAdapterWrapFunctionReturnError) { using FunctionAdapter = BinaryFunctionAdapter; std::unique_ptr wrapped = - FunctionAdapter::WrapFunction([](int64_t x, uint64_t y) -> Value { - return ErrorValue(absl::InvalidArgumentError("test_error")); + FunctionAdapter::WrapFunction([this](int64_t x, uint64_t y) -> Value { + return ErrorValue::From(absl::InvalidArgumentError("test_error"), + arena()); }); std::vector args{IntValue(44), UintValue(44)}; diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index f90d24f30..c5475a024 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -159,6 +159,7 @@ cc_test( "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/time", + "@com_google_protobuf//:protobuf", ], ) diff --git a/runtime/internal/convert_constant.cc b/runtime/internal/convert_constant.cc index 832b0dfcc..edeedb4f8 100644 --- a/runtime/internal/convert_constant.cc +++ b/runtime/internal/convert_constant.cc @@ -56,7 +56,7 @@ struct ConvertVisitor { } absl::StatusOr operator()(const absl::Duration duration) { if (duration >= kDurationHigh || duration <= kDurationLow) { - return ErrorValue(*DurationOverflowError()); + return ErrorValue::WrapUnsafe(DurationOverflowError()); } return UnsafeDurationValue(duration); } diff --git a/runtime/internal/function_adapter_test.cc b/runtime/internal/function_adapter_test.cc index d57620999..fed91528e 100644 --- a/runtime/internal/function_adapter_test.cc +++ b/runtime/internal/function_adapter_test.cc @@ -23,6 +23,7 @@ #include "common/kind.h" #include "common/value.h" #include "internal/testing.h" +#include "google/protobuf/arena.h" namespace cel::runtime_internal { namespace { @@ -306,7 +307,9 @@ TEST_F(AdaptedToValueVisitorTest, StatusOrError) { } TEST_F(AdaptedToValueVisitorTest, Any) { - auto handle = cel::ErrorValue(absl::InternalError("test_error")); + google::protobuf::Arena arena; + auto handle = + cel::ErrorValue::From(absl::InternalError("test_error"), &arena); ASSERT_OK_AND_ASSIGN(auto result, AdaptedToValueVisitor{}(handle)); diff --git a/runtime/internal/legacy_runtime_type_provider.cc b/runtime/internal/legacy_runtime_type_provider.cc index 9eca07652..54113113c 100644 --- a/runtime/internal/legacy_runtime_type_provider.cc +++ b/runtime/internal/legacy_runtime_type_provider.cc @@ -51,14 +51,14 @@ class LegacyValueBuilder final : public cel::ValueBuilder { absl::StatusOr Build() && override { CEL_ASSIGN_OR_RETURN(auto value, std::move(*builder_).Build(), - _.With(cel::ErrorValueReturn())); + _.With(cel::ErrorValueReturn(arena_))); if (value.Is()) { // Make the value behave like a legacy message. Minimizes further // legacy/modern conversions (e.g. on return and when accessing fields). CEL_ASSIGN_OR_RETURN(auto legacy_value, LegacyValue(arena_, value), - _.With(cel::ErrorValueReturn())); + _.With(cel::ErrorValueReturn(arena_))); CEL_ASSIGN_OR_RETURN(auto result, ModernValue(arena_, legacy_value), - _.With(cel::ErrorValueReturn())); + _.With(cel::ErrorValueReturn(arena_))); return result; } return value; diff --git a/runtime/optional_types_test.cc b/runtime/optional_types_test.cc index 455e51988..695f8900f 100644 --- a/runtime/optional_types_test.cc +++ b/runtime/optional_types_test.cc @@ -310,7 +310,7 @@ class UnreachableFunction final : public cel::Function { absl::StatusOr Invoke(absl::Span args, const InvokeContext& context) const override { ++(*count_); - return ErrorValue(absl::CancelledError()); + return ErrorValue::From(absl::CancelledError(), context.arena()); } private: diff --git a/runtime/standard/BUILD b/runtime/standard/BUILD index 36eb2152b..d08ac2116 100644 --- a/runtime/standard/BUILD +++ b/runtime/standard/BUILD @@ -377,6 +377,7 @@ cc_library( "//common:value", "//internal:re2_options", "//internal:status_macros", + "//runtime:function", "//runtime:function_registry", "//runtime:runtime_options", "@com_google_absl//absl/status", diff --git a/runtime/standard/equality_functions.cc b/runtime/standard/equality_functions.cc index 315413ea8..03e29223f 100644 --- a/runtime/standard/equality_functions.cc +++ b/runtime/standard/equality_functions.cc @@ -169,7 +169,7 @@ absl::StatusOr> OpaqueEqual( if (auto bool_value = result.AsBool(); bool_value) { return bool_value->NativeValue(); } - return TypeConversionError(result.GetTypeName(), "bool").NativeValue(); + return common_internal::MakeTypeConversionError(result.GetTypeName(), "bool"); } absl::optional NumberFromValue(const Value& value) { diff --git a/runtime/standard/logical_functions_test.cc b/runtime/standard/logical_functions_test.cc index de50f5312..6e106cb62 100644 --- a/runtime/standard/logical_functions_test.cc +++ b/runtime/standard/logical_functions_test.cc @@ -168,9 +168,7 @@ INSTANTIATE_TEST_SUITE_P( []() -> std::vector { return {BoolValue(false)}; }, IsBool(false)}, TestCase{builtin::kNotStrictlyFalse, - []() -> std::vector { - return {ErrorValue(absl::InternalError("test"))}; - }, + []() -> std::vector { return {ErrorValue()}; }, IsBool(true)}, TestCase{builtin::kNotStrictlyFalse, []() -> std::vector { return {UnknownValue()}; }, diff --git a/runtime/standard/regex_functions.cc b/runtime/standard/regex_functions.cc index 6833f7804..e51ef821f 100644 --- a/runtime/standard/regex_functions.cc +++ b/runtime/standard/regex_functions.cc @@ -20,6 +20,7 @@ #include "common/value.h" #include "internal/re2_options.h" #include "internal/status_macros.h" +#include "runtime/function.h" #include "runtime/function_registry.h" #include "runtime/runtime_options.h" #include "re2/re2.h" @@ -32,10 +33,11 @@ absl::Status RegisterRegexFunctions(FunctionRegistry& registry, if (options.enable_regex) { auto regex_matches = [max_size = options.regex_max_program_size]( const StringValue& target, - const StringValue& regex) -> Value { + const StringValue& regex, + const Function::InvokeContext& context) -> Value { RE2 re2(regex.ToString(), cel::internal::MakeRE2Options()); CEL_RETURN_IF_ERROR(cel::internal::CheckRE2(re2, max_size)) - .With(ErrorValueReturn()); + .With(ErrorValueReturn(context.arena())); return BoolValue(RE2::PartialMatch(target.ToString(), re2)); }; From 68a6d01a9d67f2364e979740c6f97c4baf827a44 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 1 Oct 2026 16:27:21 -0700 Subject: [PATCH 256/350] Tune inlining for core interpreter loop. PiperOrigin-RevId: 991971585 --- eval/eval/equality_steps.h | 3 +- eval/eval/evaluator_core.cc | 232 +++++++---------------------------- eval/eval/evaluator_core.h | 233 ++++++++++++++++++++++++++++++++---- 3 files changed, 253 insertions(+), 215 deletions(-) diff --git a/eval/eval/equality_steps.h b/eval/eval/equality_steps.h index ef2c4bd20..141e571d1 100644 --- a/eval/eval/equality_steps.h +++ b/eval/eval/equality_steps.h @@ -19,10 +19,11 @@ #include #include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { +class ExecutionFrame; + // Factory method for recursive _==_/_!=_ Execution step std::unique_ptr CreateDirectEqualityStep( std::unique_ptr lhs, diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 77f0bb30f..b34c09c7c 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "absl/base/attributes.h" @@ -28,6 +29,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" +#include "absl/types/span.h" #include "common/value.h" #include "common/value_kind.h" #include "common/values/list_value_builder.h" @@ -52,7 +54,7 @@ void FlatExpressionEvaluatorState::Reset() { comprehension_slots_.Reset(); } -const ExpressionStep* ExecutionFrame::Next() { +const ExpressionStep* ExecutionFrame::NextSlow() { while (true) { const size_t end_pos = execution_path_.size(); @@ -79,6 +81,13 @@ const ExpressionStep* ExecutionFrame::Next() { } } +void ExecutionFrame::AbortJumpOutOfRange(int offset) { + Abort(absl::Status( + absl::StatusCode::kInternal, + absl::StrCat("Jump address out of range: position: ", pc_, + ", offset: ", offset, ", range: ", execution_path_.size()))); +} + namespace { // This class abuses the fact that `absl::Status` is trivially destructible when @@ -114,64 +123,9 @@ class EvaluationStatus final { alignas(absl::Status) char status_[sizeof(absl::Status)]; }; -void EvaluateReadSlotStep(size_t slot_index, ExecutionFrame& frame) { - const ComprehensionSlots::Slot* slot = - frame.comprehension_slots().Get(slot_index); - if (!slot->Has()) { - frame.Abort(absl::InternalError(absl::StrCat( - "Comprehension variable read out of scope: ", slot_index))); - return; - } - frame.value_stack().Push(slot->value(), slot->attribute()); -} - -void EvaluateBoolJumpStep(const BoolJumpStepInfo& step, bool target, - ExecutionFrame& frame) { - ABSL_DCHECK(step.set) << "BoolJumpStep did not have a value set."; - if (!frame.value_stack().HasEnough(step.arg_count)) { - frame.Abort( - absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); - return; - } - const cel::Value& value = frame.value_stack().Peek(); - if (value.IsBool() && value.GetBool().NativeValue() == target) { - frame.value_stack().SwapAndPop(step.arg_count, step.arg_count - 1); - frame.JumpToOrAbort(step.offset); - } - // No-op if the value is not a bool or the value is not the target. - // Cleanup will happen if we hit a later jump or we fall-through. -} - -void EvaluateTernaryJumpStep(const TernaryJumpStepInfo& step, - ExecutionFrame& frame) { - ABSL_DCHECK(step.set) << "TernaryJumpStep did not have a value set."; - if (!frame.value_stack().HasEnough(1)) { - frame.Abort(absl::InternalError("TernaryJumpStep: value stack underflow")); - return; - } - const cel::Value& condition = frame.value_stack().Peek(); - switch (condition.kind()) { - case cel::ValueKind::kBool: - if (!condition.GetBool().NativeValue()) { - frame.JumpToOrAbort(step.jump_to_second_offset); - } - frame.value_stack().Pop(); - break; - default: - frame.value_stack().PopAndPush(cel::ErrorValue::From( - cel::runtime_internal::CreateNoMatchingOverloadError( - ""), - frame.arena())); - ABSL_FALLTHROUGH_INTENDED; - case cel::ValueKind::kError: - case cel::ValueKind::kUnknown: - // Propagate the error or unknown set. - frame.JumpToOrAbort(step.error_offset); - break; - } -} +} // namespace -void EvaluateMutableListAppendStep(ExecutionFrame& frame) { +void ExpressionStep::EvaluateMutableListAppendStep(ExecutionFrame& frame) { if (!frame.value_stack().HasEnough(2)) { frame.Abort( absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); @@ -187,7 +141,7 @@ void EvaluateMutableListAppendStep(ExecutionFrame& frame) { return; } if (frame.unknown_processing_enabled()) { - absl::optional unknown_set = + std::optional unknown_set = frame.attribute_utility().IdentifyAndMergeUnknowns( args, frame.value_stack().GetAttributeSpan(2), /*use_partial=*/true); @@ -211,114 +165,33 @@ void EvaluateMutableListAppendStep(ExecutionFrame& frame) { absl::InvalidArgumentError("Unexpected call to runtime list append.")); } -} // namespace +void ExecutionFrame::Interpret() { + for (const ExpressionStep* expr = Next(); expr != nullptr; expr = Next()) { + expr->Evaluate(*this); + } +} -void ExpressionStep::Evaluate(ExecutionFrame* context) const { - switch (header_.kind) { - case ExpressionStepKind::kGenericLogic: - u_.logic->Evaluate(context); - break; - case ExpressionStepKind::kIntConstant: - context->value_stack().Push(cel::IntValue(u_.int_val)); - break; - case ExpressionStepKind::kBoolConstant: - context->value_stack().Push(cel::BoolValue(u_.bool_val)); - break; - case ExpressionStepKind::kDoubleConstant: - context->value_stack().Push(cel::DoubleValue(u_.double_val)); - break; - case ExpressionStepKind::kNullConstant: - context->value_stack().Push(cel::NullValue()); - break; - case ExpressionStepKind::kUintConstant: - context->value_stack().Push(cel::UintValue(u_.uint_val)); - break; - case ExpressionStepKind::kOtherConstant: - context->value_stack().Push(*u_.other_val); - break; - case ExpressionStepKind::kLazyInit: - EvaluateLazyInitStep(u_.lazy_init, *context); - break; - case ExpressionStepKind::kAssignSlotAndPop: - EvaluateAssignSlotAndPop(u_.slot_index, *context); - break; - case ExpressionStepKind::kClearSlots: - EvaluateClearSlotStep(u_.clear_slots, *context); - break; - case ExpressionStepKind::kBooleanNot: - EvaluateNotStep(*context); - break; - case ExpressionStepKind::kNotStrictlyFalse: - EvaluateNotStrictlyFalseStep(*context); - break; - case ExpressionStepKind::kBooleanOr: - EvaluateBoolLogicStep(BoolLogicKind::kOr, u_.arg_count, *context); - break; - case ExpressionStepKind::kBooleanAnd: - EvaluateBoolLogicStep(BoolLogicKind::kAnd, u_.arg_count, *context); - break; - case ExpressionStepKind::kComprehensionFinish: - EvaluateComprehensionFinishStep(u_.slot_index, *context); - break; - case ExpressionStepKind::kComprehensionNext: - u_.next_step.Evaluate1(context); - break; - case ExpressionStepKind::kComprehensionNext2: - u_.next_step.Evaluate2(context); - break; - case ExpressionStepKind::kComprehensionCond: - u_.cond_step.Evaluate1(context); - break; - case ExpressionStepKind::kComprehensionCond2: - u_.cond_step.Evaluate2(context); - break; - case ExpressionStepKind::kReadSlot: - EvaluateReadSlotStep(u_.slot_index, *context); - break; - case ExpressionStepKind::kBooleanOrJump: - EvaluateBoolJumpStep(u_.bool_jump_step, true, *context); - break; - case ExpressionStepKind::kBooleanAndJump: - EvaluateBoolJumpStep(u_.bool_jump_step, false, *context); - break; - case ExpressionStepKind::kTernaryJump: - EvaluateTernaryJumpStep(u_.ternary_jump_step, *context); - break; - case ExpressionStepKind::kFixedJump: - ABSL_DCHECK(u_.fixed_jump_step.set) - << "FixedJumpStep did not have a value set."; - context->JumpToOrAbort(u_.fixed_jump_step.offset); - break; - case ExpressionStepKind::kEagerFunction: - u_.eager_function_step->Evaluate(*context); - break; - case ExpressionStepKind::kLazyFunction: - u_.lazy_function_step->Evaluate(*context); - break; - case ExpressionStepKind::kFastIn: - EvaluateFastInStep(*context); - break; - case ExpressionStepKind::kFastEqual: - EvaluateFastEqualStep(/*negation=*/false, *context); - break; - case ExpressionStepKind::kFastNotEqual: - EvaluateFastEqualStep(/*negation=*/true, *context); - break; - case ExpressionStepKind::kNewMutableList: - context->value_stack().Push(cel::CustomListValue( - cel::common_internal::NewMutableListValue(context->arena()), - context->arena())); - break; - case ExpressionStepKind::kMutableListAppend: - EvaluateMutableListAppendStep(*context); - break; - case ExpressionStepKind::kMovedFrom: - context->Abort( - absl::InternalError("ExpressionStep::Evaluate called on moved-from " - "object")); - break; - default: - ABSL_UNREACHABLE(); +void ExecutionFrame::InterpretWithCallback(EvaluationListener& listener) { + for (const ExpressionStep* expr = Next(); expr != nullptr; expr = Next()) { + expr->Evaluate(*this); + if (pc_ == 0 || !expr->comes_from_ast() || !abort_status().ok()) { + // Skip if we just started a Call, if the step doesn't map to an AST + // id, or if evaluation was aborted. + continue; + } + + if (ABSL_PREDICT_FALSE(value_stack().empty())) { + ABSL_LOG(ERROR) << "Stack is empty after a ExpressionStep.Evaluate. " + "Try to disable short-circuiting."; + continue; + } + if (EvaluationStatus status(listener(expr->id(), value_stack().Peek(), + descriptor_pool(), message_factory(), + arena())); + !status.ok()) { + Abort(std::move(status).Consume()); + return; + } } } @@ -327,32 +200,9 @@ absl::StatusOr ExecutionFrame::Evaluate( const size_t initial_stack_size = value_stack().size(); if (!listener) { - for (const ExpressionStep* expr = Next(); - ABSL_PREDICT_TRUE(expr != nullptr); expr = Next()) { - expr->Evaluate(this); - } + Interpret(); } else { - for (const ExpressionStep* expr = Next(); - ABSL_PREDICT_TRUE(expr != nullptr); expr = Next()) { - expr->Evaluate(this); - if (pc_ == 0 || !expr->comes_from_ast() || !abort_status().ok()) { - // Skip if we just started a Call or if the step doesn't map to an - // AST id. - continue; - } - - if (ABSL_PREDICT_FALSE(value_stack().empty())) { - ABSL_LOG(ERROR) << "Stack is empty after a ExpressionStep.Evaluate. " - "Try to disable short-circuiting."; - continue; - } - if (EvaluationStatus status(listener(expr->id(), value_stack().Peek(), - descriptor_pool(), message_factory(), - arena())); - !status.ok()) { - return std::move(status).Consume(); - } - } + InterpretWithCallback(listener); } if (!abort_status().ok()) { diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 8e71fcd83..f6137689c 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -22,6 +22,7 @@ #include #include +#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/log/absl_check.h" @@ -33,10 +34,13 @@ #include "base/type_provider.h" #include "common/native_type.h" #include "common/value.h" +#include "common/value_kind.h" +#include "common/values/list_value_builder.h" #include "eval/eval/attribute_utility.h" #include "eval/eval/comprehension_slots.h" #include "eval/eval/comprehension_step.h" #include "eval/eval/direct_expression_step.h" +#include "eval/eval/equality_steps.h" #include "eval/eval/evaluator_stack.h" #include "eval/eval/expression_step_logic.h" #include "eval/eval/function_step.h" @@ -45,6 +49,7 @@ #include "eval/eval/logic_step.h" #include "runtime/activation_interface.h" #include "runtime/internal/activation_attribute_matcher_access.h" +#include "runtime/internal/errors.h" #include "runtime/runtime.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -148,7 +153,9 @@ class ExpressionStep { // Returns if the execution step comes from AST. bool comes_from_ast() const { return header_.id >= 0; } - void Evaluate(ExecutionFrame* context) const; + // Evaluates this step on the given execution frame. + ABSL_ATTRIBUTE_ALWAYS_INLINE inline void Evaluate( + ExecutionFrame& frame) const; const ExpressionStepLogic* GetGenericStep() const; bool IsGenericStep() const; @@ -316,6 +323,14 @@ class ExpressionStep { } private: + static ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateReadSlotStep( + size_t slot_index, ExecutionFrame& frame); + static ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateBoolJumpStep( + const BoolJumpStepInfo& step, bool target, ExecutionFrame& frame); + static ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateTernaryJumpStep( + const TernaryJumpStepInfo& step, ExecutionFrame& frame); + static void EvaluateMutableListAppendStep(ExecutionFrame& frame); + struct Header { ExpressionStepKind kind; uint16_t reserved; @@ -686,7 +701,14 @@ class ExecutionFrame : public ExecutionFrameBase { } // Returns next expression to evaluate. - const ExpressionStep* Next(); + ABSL_ATTRIBUTE_ALWAYS_INLINE const ExpressionStep* Next() { + if (ABSL_PREDICT_TRUE(pc_ < execution_path_.size())) { + const ExpressionStep* step = &execution_path_[pc_++]; + ABSL_ASSUME(step != nullptr); + return step; + } + return NextSlow(); + } // Evaluate the execution frame to completion. absl::StatusOr Evaluate(EvaluationListener& listener); @@ -697,31 +719,14 @@ class ExecutionFrame : public ExecutionFrameBase { // // Offset applies after normal pc increment. For example, JumpTo(0) is a // no-op, JumpTo(1) skips the expected next step. - absl::Status JumpTo(int offset) { - ABSL_DCHECK_LE(offset, static_cast(execution_path_.size())); - ABSL_DCHECK_GE(offset, -static_cast(pc_)); - - int new_pc = static_cast(pc_) + offset; - if (new_pc < 0 || new_pc > static_cast(execution_path_.size())) { - return absl::Status(absl::StatusCode::kInternal, - absl::StrCat("Jump address out of range: position: ", - pc_, ", offset: ", offset, - ", range: ", execution_path_.size())); - } - pc_ = static_cast(new_pc); - return absl::OkStatus(); - } - - void JumpToOrAbort(int offset) { + ABSL_ATTRIBUTE_ALWAYS_INLINE void JumpToOrAbort(int offset) { ABSL_DCHECK_LE(offset, static_cast(execution_path_.size())); ABSL_DCHECK_GE(offset, -static_cast(pc_)); int new_pc = static_cast(pc_) + offset; - if (new_pc < 0 || new_pc > static_cast(execution_path_.size())) { - Abort(absl::Status(absl::StatusCode::kInternal, - absl::StrCat("Jump address out of range: position: ", - pc_, ", offset: ", offset, - ", range: ", execution_path_.size()))); + if (ABSL_PREDICT_FALSE(new_pc < 0 || + new_pc > static_cast(execution_path_.size()))) { + AbortJumpOutOfRange(offset); return; } pc_ = static_cast(new_pc); @@ -797,6 +802,21 @@ class ExecutionFrame : public ExecutionFrameBase { size_t expected_stack_size; }; + // Runs steps until the program completes or is aborted. + void Interpret(); + + // Same as Interpret(), but also calls `listener` with the result of each + // step that maps to an AST node. + void InterpretWithCallback(EvaluationListener& listener); + + // Slow path of Next(): returns from a completed subexpression, or returns + // nullptr at the end of the program. + ABSL_ATTRIBUTE_NOINLINE const ExpressionStep* NextSlow(); + + // Error path of JumpToOrAbort(). + ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void AbortJumpOutOfRange( + int offset); + size_t pc_; // pc_ - Program Counter. Current position on execution path. ExecutionPathView execution_path_; EvaluatorStack* absl_nonnull const value_stack_; @@ -994,6 +1014,173 @@ inline ExpressionStep& ExpressionStep::operator=(ExpressionStep&& other) { return *this; } +inline void ExpressionStep::EvaluateReadSlotStep(size_t slot_index, + ExecutionFrame& frame) { + const ComprehensionSlots::Slot* slot = + frame.comprehension_slots().Get(slot_index); + if (!slot->Has()) { + frame.Abort(absl::InternalError(absl::StrCat( + "Comprehension variable read out of scope: ", slot_index))); + return; + } + frame.value_stack().Push(slot->value(), slot->attribute()); +} + +inline void ExpressionStep::EvaluateBoolJumpStep(const BoolJumpStepInfo& step, + bool target, + ExecutionFrame& frame) { + ABSL_DCHECK(step.set) << "BoolJumpStep did not have a value set."; + if (!frame.value_stack().HasEnough(step.arg_count)) { + frame.Abort( + absl::Status(absl::StatusCode::kInternal, "Value stack underflow")); + return; + } + const cel::Value& value = frame.value_stack().Peek(); + if (value.IsBool() && value.GetBool().NativeValue() == target) { + frame.value_stack().SwapAndPop(step.arg_count, step.arg_count - 1); + frame.JumpToOrAbort(step.offset); + } + // No-op if the value is not a bool or the value is not the target. + // Cleanup will happen if we hit a later jump or we fall-through. +} + +inline void ExpressionStep::EvaluateTernaryJumpStep( + const TernaryJumpStepInfo& step, ExecutionFrame& frame) { + ABSL_DCHECK(step.set) << "TernaryJumpStep did not have a value set."; + if (!frame.value_stack().HasEnough(1)) { + frame.Abort(absl::InternalError("TernaryJumpStep: value stack underflow")); + return; + } + const cel::Value& condition = frame.value_stack().Peek(); + switch (condition.kind()) { + case cel::ValueKind::kBool: + if (!condition.GetBool().NativeValue()) { + frame.JumpToOrAbort(step.jump_to_second_offset); + } + frame.value_stack().Pop(); + break; + default: + frame.value_stack().PopAndPush(cel::ErrorValue::From( + cel::runtime_internal::CreateNoMatchingOverloadError( + ""), + frame.arena())); + ABSL_FALLTHROUGH_INTENDED; + case cel::ValueKind::kError: + case cel::ValueKind::kUnknown: + // Propagate the error or unknown set. + frame.JumpToOrAbort(step.error_offset); + break; + } +} + +inline void ExpressionStep::Evaluate(ExecutionFrame& frame) const { + switch (header_.kind) { + case ExpressionStepKind::kGenericLogic: + u_.logic->Evaluate(&frame); + break; + case ExpressionStepKind::kIntConstant: + frame.value_stack().Push(cel::IntValue(u_.int_val)); + break; + case ExpressionStepKind::kBoolConstant: + frame.value_stack().Push(cel::BoolValue(u_.bool_val)); + break; + case ExpressionStepKind::kDoubleConstant: + frame.value_stack().Push(cel::DoubleValue(u_.double_val)); + break; + case ExpressionStepKind::kNullConstant: + frame.value_stack().Push(cel::NullValue()); + break; + case ExpressionStepKind::kUintConstant: + frame.value_stack().Push(cel::UintValue(u_.uint_val)); + break; + case ExpressionStepKind::kOtherConstant: + frame.value_stack().Push(*u_.other_val); + break; + case ExpressionStepKind::kLazyInit: + EvaluateLazyInitStep(u_.lazy_init, frame); + break; + case ExpressionStepKind::kAssignSlotAndPop: + EvaluateAssignSlotAndPop(u_.slot_index, frame); + break; + case ExpressionStepKind::kClearSlots: + EvaluateClearSlotStep(u_.clear_slots, frame); + break; + case ExpressionStepKind::kBooleanNot: + EvaluateNotStep(frame); + break; + case ExpressionStepKind::kNotStrictlyFalse: + EvaluateNotStrictlyFalseStep(frame); + break; + case ExpressionStepKind::kBooleanOr: + EvaluateBoolLogicStep(BoolLogicKind::kOr, u_.arg_count, frame); + break; + case ExpressionStepKind::kBooleanAnd: + EvaluateBoolLogicStep(BoolLogicKind::kAnd, u_.arg_count, frame); + break; + case ExpressionStepKind::kComprehensionFinish: + EvaluateComprehensionFinishStep(u_.slot_index, frame); + break; + case ExpressionStepKind::kComprehensionNext: + u_.next_step.Evaluate1(&frame); + break; + case ExpressionStepKind::kComprehensionNext2: + u_.next_step.Evaluate2(&frame); + break; + case ExpressionStepKind::kComprehensionCond: + u_.cond_step.Evaluate1(&frame); + break; + case ExpressionStepKind::kComprehensionCond2: + u_.cond_step.Evaluate2(&frame); + break; + case ExpressionStepKind::kReadSlot: + EvaluateReadSlotStep(u_.slot_index, frame); + break; + case ExpressionStepKind::kBooleanOrJump: + EvaluateBoolJumpStep(u_.bool_jump_step, true, frame); + break; + case ExpressionStepKind::kBooleanAndJump: + EvaluateBoolJumpStep(u_.bool_jump_step, false, frame); + break; + case ExpressionStepKind::kTernaryJump: + EvaluateTernaryJumpStep(u_.ternary_jump_step, frame); + break; + case ExpressionStepKind::kFixedJump: + ABSL_DCHECK(u_.fixed_jump_step.set) + << "FixedJumpStep did not have a value set."; + frame.JumpToOrAbort(u_.fixed_jump_step.offset); + break; + case ExpressionStepKind::kEagerFunction: + u_.eager_function_step->Evaluate(frame); + break; + case ExpressionStepKind::kLazyFunction: + u_.lazy_function_step->Evaluate(frame); + break; + case ExpressionStepKind::kFastIn: + EvaluateFastInStep(frame); + break; + case ExpressionStepKind::kFastEqual: + EvaluateFastEqualStep(/*negation=*/false, frame); + break; + case ExpressionStepKind::kFastNotEqual: + EvaluateFastEqualStep(/*negation=*/true, frame); + break; + case ExpressionStepKind::kNewMutableList: + frame.value_stack().Push(cel::CustomListValue( + cel::common_internal::NewMutableListValue(frame.arena()), + frame.arena())); + break; + case ExpressionStepKind::kMutableListAppend: + EvaluateMutableListAppendStep(frame); + break; + case ExpressionStepKind::kMovedFrom: + frame.Abort(absl::InternalError( + "ExpressionStep::Evaluate called on moved-from step")); + break; + default: + ABSL_UNREACHABLE(); + } +} + } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_EVALUATOR_CORE_H_ From 2c2a9c243aa1860ce881cb345eae42e0b89cdf28 Mon Sep 17 00:00:00 2001 From: Justin King Date: Thu, 1 Oct 2026 16:38:44 -0700 Subject: [PATCH 257/350] Optimize `BytesValueInputStream` and `BytesValueOutputStream` by avoiding `std::visit` PiperOrigin-RevId: 991977627 --- common/values/bytes_value_input_stream.h | 99 +++++------------- common/values/bytes_value_output_stream.h | 119 ++++++---------------- common/values/bytes_value_test.cc | 25 ++++- 3 files changed, 77 insertions(+), 166 deletions(-) diff --git a/common/values/bytes_value_input_stream.h b/common/values/bytes_value_input_stream.h index 35050d2de..fc2972a05 100644 --- a/common/values/bytes_value_input_stream.h +++ b/common/values/bytes_value_input_stream.h @@ -21,15 +21,12 @@ #include #include #include -#include +#include +#include -#include "absl/base/attributes.h" -#include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/variant.h" -#include "absl/utility/utility.h" #include "common/internal/byte_string.h" #include "common/values/bytes_value.h" #include "google/protobuf/io/zero_copy_stream.h" @@ -39,80 +36,44 @@ namespace cel { class BytesValueInputStream final : public google::protobuf::io::ZeroCopyInputStream { public: - explicit BytesValueInputStream( - const BytesValue* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND) { - Construct(value); - } - - ~BytesValueInputStream() override { AsVariant().~variant(); } + explicit BytesValueInputStream(const BytesValue& value) { Construct(value); } bool Next(const void** data, int* size) override { - return absl::visit( - [&data, &size](auto& alternative) -> bool { - return alternative.stream.Next(data, size); - }, - AsVariant()); + return stream_->Next(data, size); } - void BackUp(int count) override { - absl::visit( - [&count](auto& alternative) -> void { - alternative.stream.BackUp(count); - }, - AsVariant()); - } + void BackUp(int count) override { stream_->BackUp(count); } - bool Skip(int count) override { - return absl::visit( - [&count](auto& alternative) -> bool { - return alternative.stream.Skip(count); - }, - AsVariant()); - } + bool Skip(int count) override { return stream_->Skip(count); } - int64_t ByteCount() const override { - return absl::visit( - [](const auto& alternative) -> int64_t { - return alternative.stream.ByteCount(); - }, - AsVariant()); - } + int64_t ByteCount() const override { return stream_->ByteCount(); } bool ReadCord(absl::Cord* cord, int count) override { - return absl::visit( - [&cord, &count](auto& alternative) -> bool { - return alternative.stream.ReadCord(cord, count); - }, - AsVariant()); + return stream_->ReadCord(cord, count); } private: - struct ArrayStream { - ArrayStream(const char* data, int size) : stream(data, size) {} - - google::protobuf::io::ArrayInputStream stream; - }; + using ArrayStream = google::protobuf::io::ArrayInputStream; struct CordStream { - explicit CordStream(const absl::Cord& cord) - : cord(cord), stream(&this->cord) {} + explicit CordStream(absl::Cord cord) + : cord(std::move(cord)), stream(&this->cord) {} absl::Cord cord; google::protobuf::io::CordInputStream stream; }; - using Variant = absl::variant; + using Variant = std::variant; - void Construct(const BytesValue* absl_nonnull value) { - ABSL_DCHECK(value != nullptr); - - switch (value->value_.GetKind()) { + void Construct(const BytesValue& value) { + switch (value.value_.GetKind()) { case common_internal::ByteStringKind::kSmall: - Construct(value->value_.GetSmall()); + small_ = value.value_.rep_.small; + Construct(absl::string_view(small_.data, small_.size)); break; case common_internal::ByteStringKind::kMedium: - Construct(value->value_.GetMedium()); + Construct(value.value_.GetMedium()); break; case common_internal::ByteStringKind::kLarge: - Construct(value->value_.GetLarge()); + Construct(value.value_.GetLarge()); break; } } @@ -120,27 +81,17 @@ class BytesValueInputStream final : public google::protobuf::io::ZeroCopyInputSt void Construct(absl::string_view value) { ABSL_DCHECK_LE(value.size(), static_cast(std::numeric_limits::max())); - ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value.data(), - static_cast(value.size())); - } - - void Construct(const absl::Cord& value) { - ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value); - } - - void Destruct() { AsVariant().~variant(); } - - Variant& AsVariant() ABSL_ATTRIBUTE_LIFETIME_BOUND { - return *std::launder(reinterpret_cast(&impl_[0])); + stream_ = &variant_.emplace(value.data(), + static_cast(value.size())); } - const Variant& AsVariant() const ABSL_ATTRIBUTE_LIFETIME_BOUND { - return *std::launder(reinterpret_cast(&impl_[0])); + void Construct(absl::Cord value) { + stream_ = &variant_.emplace(std::move(value)).stream; } - alignas(Variant) char impl_[sizeof(Variant)]; + google::protobuf::io::ZeroCopyInputStream* stream_; + common_internal::SmallByteStringRep small_; + Variant variant_; }; } // namespace cel diff --git a/common/values/bytes_value_output_stream.h b/common/values/bytes_value_output_stream.h index b23d1bd3d..5ec3b7ea7 100644 --- a/common/values/bytes_value_output_stream.h +++ b/common/values/bytes_value_output_stream.h @@ -19,18 +19,16 @@ #define THIRD_PARTY_CEL_CPP_COMMON_VALUES_BYTES_VALUE_OUTPUT_STREAM_H_ #include -#include #include #include +#include -#include "absl/base/attributes.h" #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/functional/overload.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" -#include "absl/types/variant.h" -#include "absl/utility/utility.h" #include "common/internal/byte_string.h" #include "common/values/bytes_value.h" #include "google/protobuf/arena.h" @@ -41,99 +39,55 @@ namespace cel { class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutputStream { public: + BytesValueOutputStream() { Construct(); } + explicit BytesValueOutputStream(const BytesValue& value) { Construct(value); } bool Next(void** data, int* size) override { - return absl::visit(absl::Overload( - [&data, &size](String& string) -> bool { - return string.stream.Next(data, size); - }, - [&data, &size](Cord& cord) -> bool { - return cord.stream.Next(data, size); - }), - AsVariant()); + return stream_->Next(data, size); } - void BackUp(int count) override { - absl::visit( - absl::Overload( - [&count](String& string) -> void { string.stream.BackUp(count); }, - [&count](Cord& cord) -> void { cord.stream.BackUp(count); }), - AsVariant()); - } + void BackUp(int count) override { stream_->BackUp(count); } - int64_t ByteCount() const override { - return absl::visit(absl::Overload( - [](const String& string) -> int64_t { - return string.stream.ByteCount(); - }, - [](const Cord& cord) -> int64_t { - return cord.stream.ByteCount(); - }), - AsVariant()); - } + int64_t ByteCount() const override { return stream_->ByteCount(); } bool WriteAliasedRaw(const void* data, int size) override { - return absl::visit(absl::Overload( - [&data, &size](String& string) -> bool { - return string.stream.WriteAliasedRaw(data, size); - }, - [&data, &size](Cord& cord) -> bool { - return cord.stream.WriteAliasedRaw(data, size); - }), - AsVariant()); + return stream_->WriteAliasedRaw(data, size); } - bool AllowsAliasing() const override { - return absl::visit(absl::Overload( - [](const String& string) -> bool { - return string.stream.AllowsAliasing(); - }, - [](const Cord& cord) -> bool { - return cord.stream.AllowsAliasing(); - }), - AsVariant()); - } + bool AllowsAliasing() const override { return stream_->AllowsAliasing(); } bool WriteCord(const absl::Cord& out) override { - return absl::visit( - absl::Overload( - [&out](String& string) -> bool { - return string.stream.WriteCord(out); - }, - [&out](Cord& cord) -> bool { return cord.stream.WriteCord(out); }), - AsVariant()); + return stream_->WriteCord(out); } + [[nodiscard]] BytesValue Consume(google::protobuf::Arena* absl_nonnull arena) && { ABSL_DCHECK(arena != nullptr); - return absl::visit( - absl::Overload( - [arena](String& string) -> BytesValue { - return BytesValue::From(std::move(string.target), arena); - }, - [arena](Cord& cord) -> BytesValue { - return BytesValue::From(cord.stream.Consume(), arena); - }), - AsVariant()); + return std::visit( + absl::Overload([](std::monostate) -> BytesValue { ABSL_UNREACHABLE(); }, + [arena](StringStream& stream) -> BytesValue { + return BytesValue::From(std::move(stream.target), + arena); + }, + [arena](CordStream& stream) -> BytesValue { + return BytesValue::From(stream.Consume(), arena); + }), + variant_); } private: - struct String final { - explicit String(absl::string_view target) + struct StringStream final { + explicit StringStream(absl::string_view target) : target(target), stream(&this->target) {} std::string target; google::protobuf::io::StringOutputStream stream; }; + using CordStream = google::protobuf::io::CordOutputStream; + using Variant = std::variant; - struct Cord final { - explicit Cord(const absl::Cord& cord) : stream(cord) {} - - google::protobuf::io::CordOutputStream stream; - }; - - using Variant = absl::variant; + void Construct() { stream_ = &variant_.emplace(); } void Construct(const BytesValue& value) { switch (value.value_.GetKind()) { @@ -150,26 +104,15 @@ class BytesValueOutputStream final : public google::protobuf::io::ZeroCopyOutput } void Construct(absl::string_view value) { - ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value); - } - - void Construct(const absl::Cord& value) { - ::new (static_cast(&impl_[0])) - Variant(absl::in_place_type, value); - } - - void Destruct() { AsVariant().~variant(); } - - Variant& AsVariant() ABSL_ATTRIBUTE_LIFETIME_BOUND { - return *std::launder(reinterpret_cast(&impl_[0])); + stream_ = &variant_.emplace(value).stream; } - const Variant& AsVariant() const ABSL_ATTRIBUTE_LIFETIME_BOUND { - return *std::launder(reinterpret_cast(&impl_[0])); + void Construct(absl::Cord value) { + stream_ = &variant_.emplace(std::move(value)); } - alignas(Variant) char impl_[sizeof(Variant)]; + google::protobuf::io::ZeroCopyOutputStream* stream_; + Variant variant_; }; } // namespace cel diff --git a/common/values/bytes_value_test.cc b/common/values/bytes_value_test.cc index e4e7ad665..d2d62b5af 100644 --- a/common/values/bytes_value_test.cc +++ b/common/values/bytes_value_test.cc @@ -168,9 +168,25 @@ TEST_F(BytesValueTest, Comparison) { EXPECT_FALSE(BytesValue::WrapUnsafe("foo") < BytesValue::WrapUnsafe("bar")); } -TEST_F(BytesValueTest, StringInputStream) { +TEST_F(BytesValueTest, SmallStringInputStream) { + BytesValue value = BytesValue::From("foo", arena()); + BytesValueInputStream stream(value); + const void* data; + int size; + absl::Cord cord; + ASSERT_TRUE(stream.Next(&data, &size)); + EXPECT_THAT(data, NotNull()); + EXPECT_EQ(size, 3); + EXPECT_EQ(stream.ByteCount(), 3); + stream.BackUp(size); + ASSERT_TRUE(stream.Skip(3)); + EXPECT_FALSE(stream.ReadCord(&cord, 3)); + EXPECT_FALSE(stream.Next(&data, &size)); +} + +TEST_F(BytesValueTest, MediumStringInputStream) { BytesValue value = BytesValue::WrapUnsafe("foo"); - BytesValueInputStream stream(&value); + BytesValueInputStream stream(value); const void* data; int size; absl::Cord cord; @@ -185,8 +201,9 @@ TEST_F(BytesValueTest, StringInputStream) { } TEST_F(BytesValueTest, CordInputStream) { - BytesValue value = BytesValue::From(absl::Cord("foo"), arena()); - BytesValueInputStream stream(&value); + absl::Cord value_cord("foo"); + BytesValue value = BytesValue::WrapUnsafe(&value_cord); + BytesValueInputStream stream(value); const void* data; int size; absl::Cord cord; From e9f491e33c3b1fb18d1f7abceba185db7b4ee096 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Thu, 1 Oct 2026 16:57:43 -0700 Subject: [PATCH 258/350] Add special case for identifier step. PiperOrigin-RevId: 991987791 --- eval/compiler/flat_expr_builder.cc | 2 +- eval/eval/BUILD | 20 ++---------- eval/eval/comprehension_step_test.cc | 6 ++-- eval/eval/container_access_step_test.cc | 5 ++- eval/eval/create_list_step_test.cc | 2 +- eval/eval/create_map_step_test.cc | 7 ++--- eval/eval/create_struct_step_test.cc | 4 +-- eval/eval/evaluator_core.h | 37 +++++++++++++++++----- eval/eval/function_step_test.cc | 3 +- eval/eval/ident_step.cc | 41 ++++++++----------------- eval/eval/ident_step.h | 6 ++-- eval/eval/ident_step_test.cc | 20 +++--------- eval/eval/logic_step_test.cc | 4 +-- eval/eval/select_step_test.cc | 30 ++++++------------ eval/eval/ternary_step_test.cc | 6 ++-- eval/public/BUILD | 2 +- 16 files changed, 79 insertions(+), 116 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 7b9f6dabf..5c8d7c36a 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -916,7 +916,7 @@ class FlatExprVisitor : public cel::AstVisitor { if (options_.max_recursion_depth != 0) { SetRecursiveStep(CreateDirectIdentStep(ident_name, expr.id()), 1); } else { - AddStep(CreateIdentStep(ident_name), expr.id()); + AddStep(ExpressionStep::MakeIdentifierStep(ident_name, expr.id())); } } diff --git a/eval/eval/BUILD b/eval/eval/BUILD index d108aa8de..45fa550d5 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -39,6 +39,7 @@ cc_library( "equality_steps.cc", "evaluator_core.cc", "function_step.cc", + "ident_step.cc", "lazy_init_step.cc", "logic_step.cc", ], @@ -47,6 +48,7 @@ cc_library( "equality_steps.h", "evaluator_core.h", "function_step.h", + "ident_step.h", "lazy_init_step.h", "logic_step.h", ], @@ -288,26 +290,8 @@ cc_library( cc_library( name = "ident_step", - srcs = [ - "ident_step.cc", - ], - hdrs = [ - "ident_step.h", - ], deps = [ - ":attribute_trail", - ":comprehension_slots", - ":direct_expression_step", ":evaluator_core", - ":expression_step_base", - ":expression_step_logic", - "//common:value", - "//eval/internal:errors", - "//internal:status_macros", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", ], ) diff --git a/eval/eval/comprehension_step_test.cc b/eval/eval/comprehension_step_test.cc index 2ffbafb1b..7725c4a73 100644 --- a/eval/eval/comprehension_step_test.cc +++ b/eval/eval/comprehension_step_test.cc @@ -94,7 +94,7 @@ MATCHER_P(CelStringValue, val, "") { TEST_F(ListKeysStepTest, MapPartiallyUnknown) { ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); + path.push_back(ExpressionStep::MakeIdentifierStep("var")); auto init_step = std::make_unique(/*iter_slot=*/0, /*accu_slot=*/0); init_step->set_error_jump_offset(1); @@ -132,7 +132,7 @@ TEST_F(ListKeysStepTest, MapPartiallyUnknown) { TEST_F(ListKeysStepTest, ErrorPassedThrough) { ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); + path.push_back(ExpressionStep::MakeIdentifierStep("var")); auto init_step = std::make_unique(/*iter_slot=*/0, /*accu_slot=*/0); init_step->set_error_jump_offset(1); @@ -157,7 +157,7 @@ TEST_F(ListKeysStepTest, ErrorPassedThrough) { TEST_F(ListKeysStepTest, UnknownSetPassedThrough) { ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("var"))); + path.push_back(ExpressionStep::MakeIdentifierStep("var")); auto init_step = std::make_unique(/*iter_slot=*/0, /*accu_slot=*/0); init_step->set_error_jump_offset(1); diff --git a/eval/eval/container_access_step_test.cc b/eval/eval/container_access_step_test.cc index a0a69b2ef..5001c93ef 100644 --- a/eval/eval/container_access_step_test.cc +++ b/eval/eval/container_access_step_test.cc @@ -83,9 +83,8 @@ CelValue EvaluateAttributeHelper( /*enable_optional_types=*/false, 3)), 3)); } else { - path.push_back( - ExpressionStep::MakeGenericStep(CreateIdentStep("container"), 1)); - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("key"), 2)); + path.push_back(ExpressionStep::MakeIdentifierStep("container", 1)); + path.push_back(ExpressionStep::MakeIdentifierStep("key", 2)); path.push_back(ExpressionStep::MakeGenericStep( std::move(CreateContainerAccessStep(call).value()), 3)); } diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index a495aad11..4475c1fb2 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -119,7 +119,7 @@ absl::StatusOr RunExpressionWithCelValues( expr0.set_id(ind); expr0.mutable_ident_expr().set_name(var_name); - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep(var_name))); + path.push_back(ExpressionStep::MakeIdentifierStep(var_name)); activation.InsertValue(var_name, value); } diff --git a/eval/eval/create_map_step_test.cc b/eval/eval/create_map_step_test.cc index 36a6eebd4..d231db211 100644 --- a/eval/eval/create_map_step_test.cc +++ b/eval/eval/create_map_step_test.cc @@ -71,11 +71,8 @@ absl::StatusOr CreateStackMachineProgram( std::string key_name = absl::StrCat("key", index); std::string value_name = absl::StrCat("value", index); - auto step_key = CreateIdentStep(key_name); - auto step_value = CreateIdentStep(value_name); - - path.push_back(ExpressionStep::MakeGenericStep(std::move(step_key))); - path.push_back(ExpressionStep::MakeGenericStep(std::move(step_value))); + path.push_back(ExpressionStep::MakeIdentifierStep(key_name)); + path.push_back(ExpressionStep::MakeIdentifierStep(value_name)); activation.InsertValue(key_name, item.first); activation.InsertValue(value_name, item.second); diff --git a/eval/eval/create_struct_step_test.cc b/eval/eval/create_struct_step_test.cc index 056b10267..06085c311 100644 --- a/eval/eval/create_struct_step_test.cc +++ b/eval/eval/create_struct_step_test.cc @@ -72,13 +72,11 @@ using ::testing::Pointwise; absl::StatusOr MakeStackMachinePath(absl::string_view field) { ExecutionPath path; - auto step0 = CreateIdentStep("message"); - auto step1 = CreateCreateStructStep("google.api.expr.runtime.TestMessage", {std::string(field)}, /*optional_indices=*/{}); - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0))); + path.push_back(ExpressionStep::MakeIdentifierStep("message")); path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); return path; diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index f6137689c..34db30271 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -44,6 +45,7 @@ #include "eval/eval/evaluator_stack.h" #include "eval/eval/expression_step_logic.h" #include "eval/eval/function_step.h" +#include "eval/eval/ident_step.h" #include "eval/eval/iterator_stack.h" #include "eval/eval/lazy_init_step.h" #include "eval/eval/logic_step.h" @@ -99,17 +101,19 @@ enum class ExpressionStepKind : uint16_t { kBooleanAndJump = 22, kTernaryJump = 23, kFixedJump = 24, + // Identifier + kIdentifier = 25, // Functions calls. - kEagerFunction = 25, - kLazyFunction = 26, + kEagerFunction = 26, + kLazyFunction = 27, // fast built-ins. These are used if we know they haven't been extended. // otherwise we use normal function call steps. - kFastIn = 27, - kFastEqual = 28, - kFastNotEqual = 29, + kFastIn = 28, + kFastEqual = 29, + kFastNotEqual = 30, // Special built-in steps for mutable lists implementing map/filter. - kNewMutableList = 30, - kMutableListAppend = 31, + kNewMutableList = 31, + kMutableListAppend = 32, }; struct BoolJumpStepInfo { @@ -288,6 +292,18 @@ class ExpressionStep { return step; } + static ExpressionStep MakeIdentifierStep(absl::string_view identifier, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kIdentifier, id); + step.u_.identifier = new std::string(identifier); + return step; + } + + static ExpressionStep MakeIdentStep(absl::string_view identifier, + int64_t id = -1) { + return MakeIdentifierStep(identifier, id); + } + static ExpressionStep MakeFastInStep(int64_t id = -1) { return ExpressionStep(ExpressionStepKind::kFastIn, id); } @@ -391,6 +407,7 @@ class ExpressionStep { FixedJumpStepInfo fixed_jump_step; EagerFunctionStep* eager_function_step; LazyFunctionStep* lazy_function_step; + std::string* identifier; Data() : empty(nullptr) {} ~Data() {} @@ -949,6 +966,9 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kOtherConstant: delete u_.other_val; break; + case ExpressionStepKind::kIdentifier: + delete u_.identifier; + break; case ExpressionStepKind::kEagerFunction: delete u_.eager_function_step; break; @@ -1149,6 +1169,9 @@ inline void ExpressionStep::Evaluate(ExecutionFrame& frame) const { << "FixedJumpStep did not have a value set."; frame.JumpToOrAbort(u_.fixed_jump_step.offset); break; + case ExpressionStepKind::kIdentifier: + EvaluateIdentifierStep(*u_.identifier, frame); + break; case ExpressionStepKind::kEagerFunction: u_.eager_function_step->Evaluate(frame); break; diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index 0480c1446..2946a087a 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -674,8 +674,7 @@ TEST_P(FunctionStepTestUnknowns, PartialUnknownHandlingTest) { IdentExpr ident1; ident1.set_name("param"); CallExpr call1 = SinkFunction::MakeCall(); - auto step0 = - ExpressionStep::MakeGenericStep(CreateIdentStep("param"), GetExprId()); + auto step0 = ExpressionStep::MakeIdentifierStep("param", GetExprId()); ASSERT_OK_AND_ASSIGN(auto step1, MakeTestFunctionStep(call1, registry)); path.push_back(std::move(step0)); diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index 23d1f5304..088e39bbe 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -16,7 +16,6 @@ #include "eval/eval/comprehension_slots.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" #include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" #include "internal/status_macros.h" @@ -28,16 +27,6 @@ namespace { using ::cel::Value; using ::cel::runtime_internal::CreateError; -class IdentStep : public ExpressionStepBase { - public: - explicit IdentStep(absl::string_view name) : name_(name) {} - - void Evaluate(ExecutionFrame* frame) const override; - - private: - std::string name_; -}; - absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, Value& result, AttributeTrail& attribute) { if (frame.attribute_tracking_enabled()) { @@ -74,19 +63,6 @@ absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, return absl::OkStatus(); } -void IdentStep::Evaluate(ExecutionFrame* frame) const { - Value value; - AttributeTrail attribute; - - if (absl::Status status = LookupIdent(name_, *frame, value, attribute); - !status.ok()) { - frame->Abort(std::move(status)); - return; - } - - frame->value_stack().Push(std::move(value), std::move(attribute)); -} - absl::StatusOr LookupSlot( absl::string_view name, size_t slot_index, ExecutionFrameBase& frame) { ComprehensionSlots::Slot* slot = frame.comprehension_slots().Get(slot_index); @@ -137,6 +113,18 @@ class DirectSlotStep : public DirectExpressionStep { } // namespace +void EvaluateIdentifierStep(absl::string_view identifier, + ExecutionFrame& frame) { + frame.value_stack().Push(cel::NullValue()); + if (absl::Status status = + LookupIdent(identifier, frame, frame.value_stack().Peek(), + frame.value_stack().PeekAttribute()); + !status.ok()) { + frame.Abort(std::move(status)); + return; + } +} + std::unique_ptr CreateDirectIdentStep( absl::string_view identifier, int64_t expr_id) { return std::make_unique(identifier, expr_id); @@ -147,9 +135,4 @@ std::unique_ptr CreateDirectSlotIdentStep( return std::make_unique(identifier, slot_index, expr_id); } -std::unique_ptr CreateIdentStep( - const absl::string_view name) { - return std::make_unique(name); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step.h b/eval/eval/ident_step.h index 508056827..9bafa77fe 100644 --- a/eval/eval/ident_step.h +++ b/eval/eval/ident_step.h @@ -11,14 +11,16 @@ namespace google::api::expr::runtime { +class ExecutionFrame; + std::unique_ptr CreateDirectIdentStep( absl::string_view identifier, int64_t expr_id); std::unique_ptr CreateDirectSlotIdentStep( absl::string_view identifier, size_t slot_index, int64_t expr_id); -// Factory method for Ident - based Execution step -std::unique_ptr CreateIdentStep(absl::string_view name); +void EvaluateIdentifierStep(absl::string_view identifier, + ExecutionFrame& frame); } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step_test.cc b/eval/eval/ident_step_test.cc index 7be2a32dc..91b73a0fd 100644 --- a/eval/eval/ident_step_test.cc +++ b/eval/eval/ident_step_test.cc @@ -47,10 +47,8 @@ using ::testing::HasSubstr; using ::testing::SizeIs; TEST(IdentStepTest, TestIdentStep) { - auto step = CreateIdentStep("name0"); - ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); + path.push_back(ExpressionStep::MakeIdentifierStep("name0")); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( @@ -73,10 +71,8 @@ TEST(IdentStepTest, TestIdentStep) { } TEST(IdentStepTest, TestIdentStepNameNotFound) { - auto step = CreateIdentStep("name0"); - ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); + path.push_back(ExpressionStep::MakeIdentifierStep("name0")); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl impl( @@ -96,10 +92,8 @@ TEST(IdentStepTest, TestIdentStepNameNotFound) { } TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { - auto step = CreateIdentStep("name0"); - ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); + path.push_back(ExpressionStep::MakeIdentifierStep("name0")); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kDisabled; auto env = NewTestingRuntimeEnv(); @@ -132,10 +126,8 @@ TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { } TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { - auto step = CreateIdentStep("name0"); - ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); + path.push_back(ExpressionStep::MakeIdentifierStep("name0")); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kDisabled; @@ -172,10 +164,8 @@ TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { } TEST(IdentStepTest, TestIdentStepUnknownAttribute) { - auto step = CreateIdentStep("name0"); - ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); + path.push_back(ExpressionStep::MakeIdentifierStep("name0")); // Expression with unknowns enabled. cel::RuntimeOptions options; diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index 8bd4834f9..b289e0e0a 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -67,8 +67,8 @@ class LogicStepTest : public testing::TestWithParam { absl::Status EvaluateLogic(CelValue arg0, CelValue arg1, bool is_or, CelValue* result, bool enable_unknown) { ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name0"))); - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name1"))); + path.push_back(ExpressionStep::MakeIdentifierStep("name0")); + path.push_back(ExpressionStep::MakeIdentifierStep("name1")); path.push_back( (is_or) ? ExpressionStep::MakeBooleanOrStep(/*num_args=*/2, /*id=*/2) : ExpressionStep::MakeBooleanAndStep(/*num_args=*/2, /*id=*/2)); diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index 874b5ea76..3ab35fd63 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -104,14 +104,13 @@ class SelectStepTest : public testing::Test { auto& ident = expr0.mutable_ident_expr(); ident.set_name("target"); - auto step0 = CreateIdentStep(ident.name()); CEL_ASSIGN_OR_RETURN( auto step1, CreateSelectStep(select.field(), select.test_only(), options.enable_wrapper_type_null_unboxing)); path.push_back( - ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + ExpressionStep::MakeIdentifierStep(ident.name(), expr0.id())); path.push_back( ExpressionStep::MakeGenericStep(std::move(step1), expr.id())); @@ -288,7 +287,6 @@ TEST_F(SelectStepTest, MapPresenseIsErrorTest) { Expr& expr0 = select_map.mutable_operand(); auto& ident = expr0.mutable_ident_expr(); ident.set_name("target"); - auto step0 = CreateIdentStep(ident.name()); ASSERT_OK_AND_ASSIGN( auto step1, CreateSelectStep(select_map.field(), select_map.test_only(), @@ -299,7 +297,7 @@ TEST_F(SelectStepTest, MapPresenseIsErrorTest) { /*enable_wrapper_type_null_unboxing=*/false)); ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back(ExpressionStep::MakeIdentifierStep(ident.name(), expr0.id())); path.push_back(ExpressionStep::MakeGenericStep(std::move(step1), expr1.id())); path.push_back( ExpressionStep::MakeGenericStep(std::move(step2), select_expr.id())); @@ -749,13 +747,12 @@ TEST_P(SelectStepConformanceTest, CelErrorAsArgument) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - auto step0 = CreateIdentStep(ident.name()); ASSERT_OK_AND_ASSIGN( auto step1, CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back(ExpressionStep::MakeIdentifierStep(ident.name(), expr0.id())); path.push_back( ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); @@ -791,13 +788,12 @@ TEST_F(SelectStepTest, DisableMissingAttributeOK) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - auto step0 = CreateIdentStep(ident.name()); ASSERT_OK_AND_ASSIGN( auto step1, CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back(ExpressionStep::MakeIdentifierStep(ident.name(), expr0.id())); path.push_back( ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); @@ -834,13 +830,12 @@ TEST_F(SelectStepTest, UnrecoverableUnknownValueProducesError) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - auto step0 = CreateIdentStep(ident.name()); ASSERT_OK_AND_ASSIGN( auto step1, CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false)); - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back(ExpressionStep::MakeIdentifierStep(ident.name(), expr0.id())); path.push_back( ExpressionStep::MakeGenericStep(std::move(step1), dummy_expr.id())); @@ -883,14 +878,13 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { auto& ident = expr0.mutable_ident_expr(); ident.set_name("message"); - auto step0 = CreateIdentStep(ident.name()); auto step1_status = CreateSelectStep(select.field(), select.test_only(), /*enable_wrapper_type_null_unboxing=*/false); ASSERT_THAT(step1_status, IsOk()); - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0), expr0.id())); + path.push_back(ExpressionStep::MakeIdentifierStep(ident.name(), expr0.id())); path.push_back(ExpressionStep::MakeGenericStep(std::move(*step1_status), dummy_expr.id())); @@ -977,8 +971,6 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { } TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { - auto step0 = CreateIdentStep("message"); - cel::StructType resolved_operand_type( (cel::MessageType(TestAllTypes::descriptor()))); const google::protobuf::FieldDescriptor* field_desc = @@ -994,7 +986,7 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { /*enable_optional_types=*/false)); ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0))); + path.push_back(ExpressionStep::MakeIdentifierStep("message")); path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); cel::RuntimeOptions options; if (GetParam()) { @@ -1017,8 +1009,6 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepTest) { } TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { - auto step0 = CreateIdentStep("message"); - cel::StructType resolved_operand_type( (cel::MessageType(TestAllTypes::descriptor()))); const google::protobuf::FieldDescriptor* field_desc = @@ -1034,7 +1024,7 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { /*enable_optional_types=*/false)); ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0))); + path.push_back(ExpressionStep::MakeIdentifierStep("message")); path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); cel::RuntimeOptions options; if (GetParam()) { @@ -1054,8 +1044,6 @@ TEST_P(SelectStepConformanceTest, TypedSelectStepPropagatesUnknown) { } TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { - auto step0 = CreateIdentStep("message"); - cel::StructType resolved_operand_type( (cel::MessageType(TestAllTypes::descriptor()))); const google::protobuf::FieldDescriptor* field_desc = @@ -1071,7 +1059,7 @@ TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { /*enable_optional_types=*/false)); ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(std::move(step0))); + path.push_back(ExpressionStep::MakeIdentifierStep("message")); path.push_back(ExpressionStep::MakeGenericStep(std::move(step1))); cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; diff --git a/eval/eval/ternary_step_test.cc b/eval/eval/ternary_step_test.cc index db1df46f9..cedab0ca2 100644 --- a/eval/eval/ternary_step_test.cc +++ b/eval/eval/ternary_step_test.cc @@ -65,9 +65,9 @@ class LogicStepTest : public testing::TestWithParam { CelValue* result, bool enable_unknown) { ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name0"))); - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name1"))); - path.push_back(ExpressionStep::MakeGenericStep(CreateIdentStep("name2"))); + path.push_back(ExpressionStep::MakeIdentifierStep("name0")); + path.push_back(ExpressionStep::MakeIdentifierStep("name1")); + path.push_back(ExpressionStep::MakeIdentifierStep("name2")); path.push_back(ExpressionStep::MakeGenericStep(CreateTernaryStep(), 4)); cel::RuntimeOptions options; diff --git a/eval/public/BUILD b/eval/public/BUILD index a0a3bcb27..44293e4d6 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -730,7 +730,7 @@ cc_test( ":cel_attribute", ":cel_function", "//eval/eval:attribute_trail", - "//eval/eval:ident_step", + "//eval/eval:evaluator_core", "//extensions/protobuf:memory_manager", "//internal:status_macros", "//internal:testing", From 31ebbf99e45ae375b0954426bf5fe6116f38f757 Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 2 Oct 2026 04:00:10 -0700 Subject: [PATCH 259/350] Untie `CelEvaluationState` from the arena PiperOrigin-RevId: 992230397 --- eval/eval/BUILD | 1 + eval/eval/cel_expression_flat_impl.cc | 68 +++++++++++++++++++-------- eval/eval/cel_expression_flat_impl.h | 67 ++++++++++++++------------ eval/eval/evaluator_core.cc | 8 ++++ eval/eval/evaluator_core.h | 35 ++++++++++++-- eval/public/BUILD | 3 +- eval/public/cel_expression.h | 46 ++++++++++++++---- eval/tests/mock_cel_expression.h | 13 +++++ 8 files changed, 178 insertions(+), 63 deletions(-) diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 45fa550d5..66d5092ab 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -123,6 +123,7 @@ cc_library( "//internal:status_macros", "//runtime/internal:runtime_env", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/eval/eval/cel_expression_flat_impl.cc b/eval/eval/cel_expression_flat_impl.cc index 8c78d21ce..c0f0b5e71 100644 --- a/eval/eval/cel_expression_flat_impl.cc +++ b/eval/eval/cel_expression_flat_impl.cc @@ -19,6 +19,7 @@ #include #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -74,22 +75,39 @@ CelExpressionFlatEvaluationState::CelExpressionFlatEvaluationState( : state_(expression.MakeEvaluatorState(descriptor_pool, message_factory, arena)) {} +CelExpressionFlatEvaluationState::CelExpressionFlatEvaluationState( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + const FlatExpression& expr) + : state_(expr.MakeEvaluatorState(descriptor_pool, message_factory)) {} + absl::StatusOr CelExpressionFlatImpl::Trace( - const BaseActivation& activation, CelEvaluationState* _state, - CelEvaluationListener callback) const { - auto state = - ::cel::internal::down_cast(_state); - state->state().Reset(); + const BaseActivation& activation, google::protobuf::Arena* arena, + CelEvaluationListener callback, CelEvaluationState* state) const { + std::unique_ptr inline_state; + if (state == nullptr) { + inline_state = CreateState(); + state = inline_state.get(); + } + auto derived_state = + ::cel::internal::down_cast(state); + if (arena != nullptr) { + derived_state->Rebind(arena); + } else { + arena = derived_state->arena(); + } + ABSL_DCHECK(arena != nullptr) + << "arena must be implicitly provided when using InitializeState() or " + "explicitly provided when using CreateState()"; cel::interop_internal::AdapterActivationImpl modern_activation(activation); CEL_ASSIGN_OR_RETURN(cel::Value value, flat_expression_.EvaluateWithCallback( modern_activation, /*embedder_context=*/nullptr, - AdaptListener(callback), state->state())); + AdaptListener(callback), derived_state->state())); - return cel::interop_internal::ModernValueToLegacyValueOrDie(state->arena(), - value); + return cel::interop_internal::ModernValueToLegacyValueOrDie(arena, value); } std::unique_ptr CelExpressionFlatImpl::InitializeState( @@ -99,9 +117,10 @@ std::unique_ptr CelExpressionFlatImpl::InitializeState( flat_expression_); } -absl::StatusOr CelExpressionFlatImpl::Evaluate( - const BaseActivation& activation, CelEvaluationState* state) const { - return Trace(activation, state, CelEvaluationListener()); +std::unique_ptr CelExpressionFlatImpl::CreateState() const { + return std::make_unique( + env_->descriptor_pool.get(), env_->MutableMessageFactory(), + flat_expression_); } absl::StatusOr> @@ -126,14 +145,30 @@ CelExpressionRecursiveImpl::Create( absl::StatusOr CelExpressionRecursiveImpl::Trace( const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback) const { + CelEvaluationListener callback, CelEvaluationState* state) const { + std::unique_ptr inline_state; + if (state == nullptr) { + inline_state = CreateState(); + state = inline_state.get(); + } + auto derived_state = ::cel::internal::down_cast(state); + if (arena != nullptr) { + derived_state->Rebind(arena); + } else { + arena = derived_state->arena(); + } + if (state != inline_state.get()) { + derived_state->comprehension_slots().Reset(); + } + ABSL_DCHECK(arena != nullptr) + << "arena must be implicitly provided when using InitializeState() or " + "explicitly provided when using CreateState()"; cel::interop_internal::AdapterActivationImpl modern_activation(activation); - ComprehensionSlots slots(flat_expression_.comprehension_slots_size()); ExecutionFrameBase execution_frame( modern_activation, AdaptListener(callback), flat_expression_.options(), flat_expression_.type_provider(), env_->descriptor_pool.get(), env_->MutableMessageFactory(), arena, - /*embedder_context=*/nullptr, slots); + /*embedder_context=*/nullptr, derived_state->comprehension_slots()); cel::Value result; AttributeTrail trail; @@ -142,9 +177,4 @@ absl::StatusOr CelExpressionRecursiveImpl::Trace( return cel::interop_internal::ModernValueToLegacyValueOrDie(arena, result); } -absl::StatusOr CelExpressionRecursiveImpl::Evaluate( - const BaseActivation& activation, google::protobuf::Arena* arena) const { - return Trace(activation, arena, /*callback=*/nullptr); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/cel_expression_flat_impl.h b/eval/eval/cel_expression_flat_impl.h index 3590dc788..055147477 100644 --- a/eval/eval/cel_expression_flat_impl.h +++ b/eval/eval/cel_expression_flat_impl.h @@ -15,11 +15,13 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CEL_EXPRESSION_FLAT_IMPL_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CEL_EXPRESSION_FLAT_IMPL_H_ +#include #include #include #include "absl/base/nullability.h" #include "absl/status/statusor.h" +#include "eval/eval/comprehension_slots.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/public/base_activation.h" @@ -42,9 +44,18 @@ class CelExpressionFlatEvaluationState : public CelEvaluationState { google::protobuf::MessageFactory* absl_nonnull message_factory, const FlatExpression& expr); + CelExpressionFlatEvaluationState( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nonnull message_factory, + const FlatExpression& expr); + google::protobuf::Arena* arena() { return state_.arena(); } FlatExpressionEvaluatorState& state() { return state_; } + void Rebind(google::protobuf::Arena* arena) { + state_.Rebind(arena, state_.message_factory()); + } + private: FlatExpressionEvaluatorState state_; }; @@ -70,22 +81,14 @@ class CelExpressionFlatImpl : public CelExpression { std::unique_ptr InitializeState( google::protobuf::Arena* arena) const override; - absl::StatusOr Evaluate(const BaseActivation& activation, - google::protobuf::Arena* arena) const override { - return Evaluate(activation, InitializeState(arena).get()); - } - - absl::StatusOr Evaluate(const BaseActivation& activation, - CelEvaluationState* state) const override; - absl::StatusOr Trace( - const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback) const override { - return Trace(activation, InitializeState(arena).get(), callback); - } + // Implement CelExpression. + std::unique_ptr CreateState() const override; + using CelExpression::Trace; absl::StatusOr Trace(const BaseActivation& activation, - CelEvaluationState* state, - CelEvaluationListener callback) const override; + google::protobuf::Arena* arena, + CelEvaluationListener callback, + CelEvaluationState* state) const override; // Exposed for inspection in tests. const FlatExpression& flat_expression() const { return flat_expression_; } @@ -105,11 +108,21 @@ class CelExpressionRecursiveImpl : public CelExpression { private: class EvaluationState : public CelEvaluationState { public: - explicit EvaluationState(google::protobuf::Arena* arena) : arena_(arena) {} + explicit EvaluationState(size_t comprehension_slots) + : EvaluationState(nullptr, comprehension_slots) {} + + EvaluationState(google::protobuf::Arena* arena, size_t comprehension_slots) + : arena_(arena), comprehension_slots_(comprehension_slots) {} + google::protobuf::Arena* arena() { return arena_; } + void Rebind(google::protobuf::Arena* arena) { arena_ = arena; } + + ComprehensionSlots& comprehension_slots() { return comprehension_slots_; } + private: google::protobuf::Arena* arena_; + ComprehensionSlots comprehension_slots_; }; public: @@ -127,28 +140,20 @@ class CelExpressionRecursiveImpl : public CelExpression { // Implement CelExpression. std::unique_ptr InitializeState( google::protobuf::Arena* arena) const override { - return std::make_unique(arena); + return std::make_unique( + arena, flat_expression_.comprehension_slots_size()); } - absl::StatusOr Evaluate(const BaseActivation& activation, - google::protobuf::Arena* arena) const override; - - absl::StatusOr Evaluate(const BaseActivation& activation, - CelEvaluationState* state) const override { - auto* state_impl = cel::internal::down_cast(state); - return Evaluate(activation, state_impl->arena()); + // Implement CelExpression. + std::unique_ptr CreateState() const override { + return std::make_unique( + flat_expression_.comprehension_slots_size()); } absl::StatusOr Trace(const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback) const override; - - absl::StatusOr Trace( - const BaseActivation& activation, CelEvaluationState* state, - CelEvaluationListener callback) const override { - auto* state_impl = cel::internal::down_cast(state); - return Trace(activation, state_impl->arena(), callback); - } + CelEvaluationListener callback, + CelEvaluationState* state) const override; // Exposed for inspection in tests. const FlatExpression& flat_expression() const { return flat_expression_; } diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index b34c09c7c..6d31a7b4b 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -231,6 +231,14 @@ FlatExpressionEvaluatorState FlatExpression::MakeEvaluatorState( message_factory, arena); } +FlatExpressionEvaluatorState FlatExpression::MakeEvaluatorState( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nullable message_factory) const { + return FlatExpressionEvaluatorState(path_.size(), comprehension_slots_size_, + type_provider_, descriptor_pool, + message_factory); +} + absl::StatusOr FlatExpression::EvaluateWithCallback( const cel::ActivationInterface& activation, const cel::EmbedderContext* absl_nullable embedder_context, diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 34db30271..30e2c0087 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -449,6 +449,21 @@ using ExecutionPathView = absl::Span; // evaluation. This can be reused to save on allocations. class FlatExpressionEvaluatorState { public: + FlatExpressionEvaluatorState( + size_t value_stack_size, size_t comprehension_slot_count, + const cel::TypeProvider& type_provider, + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nullable message_factory = nullptr) + : value_stack_(value_stack_size), + // We currently use comprehension_slot_count because it is less of an + // over estimate than value_stack_size. In future we should just + // calculate the correct capacity. + iterator_stack_(comprehension_slot_count), + comprehension_slots_(comprehension_slot_count), + type_provider_(type_provider), + descriptor_pool_(descriptor_pool), + message_factory_(message_factory) {} + FlatExpressionEvaluatorState( size_t value_stack_size, size_t comprehension_slot_count, const cel::TypeProvider& type_provider, @@ -483,10 +498,20 @@ class FlatExpressionEvaluatorState { } google::protobuf::MessageFactory* absl_nonnull message_factory() { + ABSL_DCHECK(message_factory_ != nullptr); return message_factory_; } - google::protobuf::Arena* absl_nonnull arena() { return arena_; } + google::protobuf::Arena* absl_nonnull arena() { + ABSL_DCHECK(arena_ != nullptr); + return arena_; + } + + void Rebind(google::protobuf::Arena* absl_nonnull arena, + google::protobuf::MessageFactory* absl_nonnull message_factory) { + arena_ = arena; + message_factory_ = message_factory; + } private: EvaluatorStack value_stack_; @@ -494,8 +519,8 @@ class FlatExpressionEvaluatorState { ComprehensionSlots comprehension_slots_; const cel::TypeProvider& type_provider_; const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool_; - google::protobuf::MessageFactory* absl_nonnull message_factory_; - google::protobuf::Arena* absl_nonnull arena_; + google::protobuf::MessageFactory* absl_nullability_unknown message_factory_ = nullptr; + google::protobuf::Arena* absl_nullability_unknown arena_ = nullptr; }; // Context needed for evaluation. This is sufficient for supporting @@ -883,6 +908,10 @@ class FlatExpression { google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const; + FlatExpressionEvaluatorState MakeEvaluatorState( + const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, + google::protobuf::MessageFactory* absl_nullable message_factory = nullptr) const; + // Evaluate the expression. // // A status may be returned if an unexpected error occurs. Recoverable errors diff --git a/eval/public/BUILD b/eval/public/BUILD index 44293e4d6..2172774a9 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -523,11 +523,12 @@ cc_library( ":cel_function_registry", ":cel_type_registry", ":cel_value", - "//common:legacy_value", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", + "@com_google_protobuf//:protobuf", ], ) diff --git a/eval/public/cel_expression.h b/eval/public/cel_expression.h index af28e2ae6..3f215207a 100644 --- a/eval/public/cel_expression.h +++ b/eval/public/cel_expression.h @@ -5,15 +5,18 @@ #include #include #include +#include #include "cel/expr/checked.pb.h" #include "cel/expr/syntax.pb.h" +#include "absl/base/attributes.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "eval/public/base_activation.h" #include "eval/public/cel_function_registry.h" #include "eval/public/cel_type_registry.h" #include "eval/public/cel_value.h" +#include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -46,34 +49,59 @@ class CelExpression { virtual ~CelExpression() = default; // Initializes the state + ABSL_DEPRECATED("Use CreateState and pass arena on each evaluation") virtual std::unique_ptr InitializeState( google::protobuf::Arena* arena) const = 0; + // Initializes the state + virtual std::unique_ptr CreateState() const = 0; + // Evaluates expression and returns value. // activation contains bindings from parameter names to values // arena parameter specifies Arena object where output result and // internal data will be allocated. virtual absl::StatusOr Evaluate(const BaseActivation& activation, - google::protobuf::Arena* arena) const = 0; + google::protobuf::Arena* arena) const { + return Evaluate(activation, arena, nullptr); + } // Evaluates expression and returns value. // activation contains bindings from parameter names to values // state must be non-null and created prior to calling Evaluate by // InitializeState. - virtual absl::StatusOr Evaluate( - const BaseActivation& activation, CelEvaluationState* state) const = 0; + ABSL_DEPRECATED( + "Use CreateState and the overloads which take both an arena and state") + virtual absl::StatusOr Evaluate(const BaseActivation& activation, + CelEvaluationState* state) const { + return Evaluate(activation, nullptr, state); + } + virtual absl::StatusOr Evaluate(const BaseActivation& activation, + google::protobuf::Arena* arena, + CelEvaluationState* state) const { + return Trace(activation, arena, nullptr, state); + } // Trace evaluates expression calling the callback on each sub-tree. - virtual absl::StatusOr Trace( - const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback) const = 0; + virtual absl::StatusOr Trace(const BaseActivation& activation, + google::protobuf::Arena* arena, + CelEvaluationListener callback) const { + return Trace(activation, arena, std::move(callback), nullptr); + } // Trace evaluates expression calling the callback on each sub-tree. // state must be non-null and created prior to calling Evaluate by // InitializeState. - virtual absl::StatusOr Trace( - const BaseActivation& activation, CelEvaluationState* state, - CelEvaluationListener callback) const = 0; + ABSL_DEPRECATED( + "Use CreateState and the overloads which take both an arena and state") + virtual absl::StatusOr Trace(const BaseActivation& activation, + CelEvaluationState* state, + CelEvaluationListener callback) const { + return Trace(activation, nullptr, std::move(callback), state); + } + virtual absl::StatusOr Trace(const BaseActivation& activation, + google::protobuf::Arena* arena, + CelEvaluationListener callback, + CelEvaluationState* state) const = 0; }; // Base class for Expression Builder implementations diff --git a/eval/tests/mock_cel_expression.h b/eval/tests/mock_cel_expression.h index 07b32b29f..4fd9c8642 100644 --- a/eval/tests/mock_cel_expression.h +++ b/eval/tests/mock_cel_expression.h @@ -15,6 +15,9 @@ class MockCelExpression : public CelExpression { MOCK_METHOD(std::unique_ptr, InitializeState, (google::protobuf::Arena * arena), (const, override)); + MOCK_METHOD(std::unique_ptr, CreateState, (), + (const, override)); + MOCK_METHOD(absl::StatusOr, Evaluate, (const BaseActivation& activation, google::protobuf::Arena* arena), (const, override)); @@ -23,6 +26,11 @@ class MockCelExpression : public CelExpression { (const BaseActivation& activation, CelEvaluationState* state), (const, override)); + MOCK_METHOD(absl::StatusOr, Evaluate, + (const BaseActivation& activation, google::protobuf::Arena* arena, + CelEvaluationState* state), + (const, override)); + MOCK_METHOD(absl::StatusOr, Trace, (const BaseActivation& activation, google::protobuf::Arena* arena, CelEvaluationListener callback), @@ -32,6 +40,11 @@ class MockCelExpression : public CelExpression { (const BaseActivation& activation, CelEvaluationState* state, CelEvaluationListener callback), (const, override)); + + MOCK_METHOD(absl::StatusOr, Trace, + (const BaseActivation& activation, google::protobuf::Arena* arena, + CelEvaluationListener callback, CelEvaluationState* state), + (const, override)); }; } // namespace google::api::expr::runtime From 9fc0f43127f188270dc9c5cbcd893ccb03a95cd3 Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 2 Oct 2026 10:26:16 -0700 Subject: [PATCH 260/350] Move `bool` back to the end of the variants PiperOrigin-RevId: 992403026 --- base/attribute.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/attribute.h b/base/attribute.h index a0877fc96..2d589368d 100644 --- a/base/attribute.h +++ b/base/attribute.h @@ -40,12 +40,12 @@ namespace common_internal { class AttributeMatcherNode; struct WildcardType {}; using AttributeQualifierVariant = - std::variant; + std::variant; using AttributeQualifierPatternVariant = - std::variant; using AttributeQualifierViewVariant = - std::variant; + std::variant; } // namespace common_internal class AttributeQualifier; From bd943b747f8de23310ddb775b92957ae2f47d638 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 4 Oct 2026 15:45:58 -0700 Subject: [PATCH 261/350] No public description PiperOrigin-RevId: 993332084 --- eval/public/cel_value.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eval/public/cel_value.h b/eval/public/cel_value.h index 76b4d09bb..5bf50d01a 100644 --- a/eval/public/cel_value.h +++ b/eval/public/cel_value.h @@ -494,7 +494,7 @@ class CelValue { explicit CelValue(T value) : value_(value) {} // Crashes with a null pointer error. - static void CrashNullPointer(Type type) ABSL_ATTRIBUTE_COLD { + ABSL_ATTRIBUTE_COLD static void CrashNullPointer(Type type) { ABSL_LOG(FATAL) << "Null pointer supplied for " << TypeName(type); // Crash ok } @@ -507,8 +507,8 @@ class CelValue { } // Crashes with a type mismatch error. - static void CrashTypeMismatch(Type requested_type, - Type actual_type) ABSL_ATTRIBUTE_COLD { + ABSL_ATTRIBUTE_COLD static void CrashTypeMismatch(Type requested_type, + Type actual_type) { ABSL_LOG(FATAL) << "Type mismatch" // Crash ok << ": expected " << TypeName(requested_type) // Crash ok << ", encountered " << TypeName(actual_type); // Crash ok From f28da962726a09a8155b4a8bd1700d910f600661 Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 5 Oct 2026 15:39:22 -0700 Subject: [PATCH 262/350] Remove `cel::Owner`, `cel::Borrower`, `cel::Unique`, `cel::Owned`, and `cel::Borrowed` smart pointers PiperOrigin-RevId: 993946502 --- common/BUILD | 21 - common/memory.h | 1251 ----------------------------- common/memory_test.cc | 466 ----------- common/value.cc | 21 +- internal/BUILD | 4 +- internal/message_equality.cc | 29 +- internal/message_equality_test.cc | 17 +- internal/well_known_types.cc | 50 +- internal/well_known_types.h | 35 +- internal/well_known_types_test.cc | 8 +- 10 files changed, 84 insertions(+), 1818 deletions(-) delete mode 100644 common/memory_test.cc diff --git a/common/BUILD b/common/BUILD index 5b6349624..f75c82f17 100644 --- a/common/BUILD +++ b/common/BUILD @@ -504,12 +504,6 @@ cc_library( hdrs = ["memory.h"], deps = [ ":allocator", - ":arena", - ":data", - ":reference_count", - "//common/internal:metadata", - "//common/internal:reference_count", - "//internal:exceptions", "//internal:to_address", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:no_destructor", @@ -520,21 +514,6 @@ cc_library( ], ) -cc_test( - name = "memory_test", - srcs = ["memory_test.cc"], - deps = [ - ":allocator", - ":data", - ":memory", - "//common/internal:reference_count", - "//internal:testing", - "@com_google_absl//absl/base:nullability", - "@com_google_protobuf//:protobuf", - "@com_google_protobuf//:struct_cc_proto", - ], -) - cc_library( name = "memory_testing", testonly = True, diff --git a/common/memory.h b/common/memory.h index 615bce519..967bc57e2 100644 --- a/common/memory.h +++ b/common/memory.h @@ -16,25 +16,16 @@ #define THIRD_PARTY_CEL_CPP_COMMON_MEMORY_H_ #include -#include #include #include #include #include #include "absl/base/attributes.h" -#include "absl/base/macros.h" #include "absl/base/nullability.h" -#include "absl/base/optimization.h" #include "absl/log/absl_check.h" #include "absl/numeric/bits.h" #include "common/allocator.h" -#include "common/arena.h" -#include "common/data.h" -#include "common/internal/metadata.h" -#include "common/internal/reference_count.h" -#include "common/reference_count.h" -#include "internal/exceptions.h" #include "internal/to_address.h" // IWYU pragma: keep #include "google/protobuf/arena.h" @@ -57,1252 +48,10 @@ enum class MemoryManagement { std::ostream& operator<<(std::ostream& out, MemoryManagement memory_management); -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Owner; -class Borrower; -template -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Unique; -template -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Owned; -template -class Borrowed; -template -struct Ownable; -template -struct Borrowable; - class MemoryManager; class ReferenceCountingMemoryManager; class PoolingMemoryManager; -namespace common_internal { -template -inline constexpr bool kNotMessageLiteAndNotData = - std::conjunction_v>, - std::negation>>; -template -inline constexpr bool kIsPointerConvertible = std::is_convertible_v; -template -inline constexpr bool kNotSameAndIsPointerConvertible = - std::conjunction_v>, - std::bool_constant>>; - -// Clears the contents of `owner`, and returns the reference count if in use. -const ReferenceCount* absl_nullable OwnerRelease(Owner owner) noexcept; -const ReferenceCount* absl_nullable BorrowerRelease(Borrower borrower) noexcept; -template -Owned WrapEternal(const T* value); - -// Pointer tag used by `cel::Unique` to indicate that the destructor needs to be -// registered with the arena, but it has not been done yet. Must be done when -// releasing. -inline constexpr uintptr_t kUniqueArenaUnownedBit = uintptr_t{1} << 0; -inline constexpr uintptr_t kUniqueArenaBits = kUniqueArenaUnownedBit; -inline constexpr uintptr_t kUniqueArenaPointerMask = ~kUniqueArenaBits; -} // namespace common_internal - -template -Owned AllocateShared(Allocator<> allocator, Args&&... args); - -template -Owned WrapShared(T* object, Allocator<> allocator); - -// `Owner` represents a reference to some co-owned data, of which this owner is -// one of the co-owners. When using reference counting, `Owner` performs -// increment/decrement where appropriate similar to `std::shared_ptr`. -// `Borrower` is similar to `Owner`, except that it is always trivially -// copyable/destructible. In that sense, `Borrower` is similar to -// `std::reference_wrapper`. -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Owner final { - private: - static constexpr uintptr_t kNone = common_internal::kMetadataOwnerNone; - static constexpr uintptr_t kReferenceCountBit = - common_internal::kMetadataOwnerReferenceCountBit; - static constexpr uintptr_t kArenaBit = - common_internal::kMetadataOwnerArenaBit; - static constexpr uintptr_t kBits = common_internal::kMetadataOwnerBits; - static constexpr uintptr_t kPointerMask = - common_internal::kMetadataOwnerPointerMask; - - public: - static Owner None() noexcept { return Owner(); } - - static Owner Allocator(Allocator<> allocator) noexcept { - auto* arena = allocator.arena(); - return arena != nullptr ? Arena(arena) : None(); - } - - static Owner Arena(google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ABSL_DCHECK(arena != nullptr); - return Owner(reinterpret_cast(arena) | kArenaBit); - } - - static Owner Arena(std::nullptr_t) = delete; - - static Owner ReferenceCount(const ReferenceCount* absl_nonnull reference_count - ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ABSL_DCHECK(reference_count != nullptr); - common_internal::StrongRef(*reference_count); - return Owner(reinterpret_cast(reference_count) | - kReferenceCountBit); - } - - static Owner ReferenceCount(std::nullptr_t) = delete; - - Owner() = default; - - Owner(const Owner& other) noexcept : Owner(CopyFrom(other.ptr_)) {} - - Owner(Owner&& other) noexcept : Owner(MoveFrom(other.ptr_)) {} - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Owner(const Owned& owned) noexcept; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Owner(Owned&& owned) noexcept; - - explicit Owner(Borrower borrower) noexcept; - - template - explicit Owner(Borrowed borrowed) noexcept; - - ~Owner() { Destroy(ptr_); } - - Owner& operator=(const Owner& other) noexcept { - if (ptr_ != other.ptr_) { - Destroy(ptr_); - ptr_ = CopyFrom(other.ptr_); - } - return *this; - } - - Owner& operator=(Owner&& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - Destroy(ptr_); - ptr_ = MoveFrom(other.ptr_); - } - return *this; - } - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Owner& operator=(const Owned& owned) noexcept; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Owner& operator=(Owned&& owned) noexcept; - - explicit operator bool() const noexcept { return !IsNone(ptr_); } - - google::protobuf::Arena* absl_nullable arena() const noexcept { - return (ptr_ & Owner::kBits) == Owner::kArenaBit - ? reinterpret_cast(ptr_ & Owner::kPointerMask) - : nullptr; - } - - void reset() noexcept { - Destroy(ptr_); - ptr_ = 0; - } - - // Tests whether two owners have ownership over the same data, that is they - // are co-owners. - friend bool operator==(const Owner& lhs, const Owner& rhs) noexcept { - // A reference count and arena can never occupy the same memory address, so - // we can compare for equality without masking off the bits. - return lhs.ptr_ == rhs.ptr_; - } - - private: - template - friend class Unique; - friend class Borrower; - template - friend Owned AllocateShared(cel::Allocator<> allocator, Args&&... args); - template - friend Owned WrapShared(T* object, cel::Allocator<> allocator); - template - friend struct Ownable; - friend const common_internal::ReferenceCount* absl_nullable - common_internal::OwnerRelease(Owner owner) noexcept; - friend const common_internal::ReferenceCount* absl_nullable - common_internal::BorrowerRelease(Borrower borrower) noexcept; - friend struct ArenaTraits; - - constexpr explicit Owner(uintptr_t ptr) noexcept : ptr_(ptr) {} - - static constexpr bool IsNone(uintptr_t ptr) noexcept { return ptr == kNone; } - - static constexpr bool IsArena(uintptr_t ptr) noexcept { - return (ptr & kArenaBit) != kNone; - } - - static constexpr bool IsReferenceCount(uintptr_t ptr) noexcept { - return (ptr & kReferenceCountBit) != kNone; - } - - ABSL_ATTRIBUTE_RETURNS_NONNULL - static google::protobuf::Arena* absl_nonnull AsArena(uintptr_t ptr) noexcept { - ABSL_ASSERT(IsArena(ptr)); - return reinterpret_cast(ptr & kPointerMask); - } - - ABSL_ATTRIBUTE_RETURNS_NONNULL - static const common_internal::ReferenceCount* absl_nonnull AsReferenceCount( - uintptr_t ptr) noexcept { - ABSL_ASSERT(IsReferenceCount(ptr)); - return reinterpret_cast( - ptr & kPointerMask); - } - - static uintptr_t CopyFrom(uintptr_t other) noexcept { return Own(other); } - - static uintptr_t MoveFrom(uintptr_t& other) noexcept { - return std::exchange(other, kNone); - } - - static void Destroy(uintptr_t ptr) noexcept { Unown(ptr); } - - static uintptr_t Own(uintptr_t ptr) noexcept { - if (IsReferenceCount(ptr)) { - const auto* refcount = Owner::AsReferenceCount(ptr); - ABSL_ASSUME(refcount != nullptr); - common_internal::StrongRef(refcount); - } - return ptr; - } - - static void Unown(uintptr_t ptr) noexcept { - if (IsReferenceCount(ptr)) { - const auto* reference_count = AsReferenceCount(ptr); - ABSL_ASSUME(reference_count != nullptr); - common_internal::StrongUnref(reference_count); - } - } - - uintptr_t ptr_ = kNone; -}; - -inline bool operator!=(const Owner& lhs, const Owner& rhs) noexcept { - return !operator==(lhs, rhs); -} - -namespace common_internal { - -inline const ReferenceCount* absl_nullable OwnerRelease(Owner owner) noexcept { - uintptr_t ptr = std::exchange(owner.ptr_, kMetadataOwnerNone); - if (Owner::IsReferenceCount(ptr)) { - return Owner::AsReferenceCount(ptr); - } - return nullptr; -} - -} // namespace common_internal - -template <> -struct ArenaTraits { - static bool trivially_destructible(const Owner& owner) { - return !Owner::IsReferenceCount(owner.ptr_); - } -}; - -// `Borrower` represents a reference to some borrowed data, where the data has -// at least one owner. When using reference counting, `Borrower` does not -// participate in incrementing/decrementing the reference count. Thus `Borrower` -// will not keep the underlying data alive. -class Borrower final { - public: - static Borrower None() noexcept { return Borrower(); } - - static Borrower Allocator(Allocator<> allocator) noexcept { - auto* arena = allocator.arena(); - return arena != nullptr ? Arena(arena) : None(); - } - - static Borrower Arena(google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ABSL_DCHECK(arena != nullptr); - return Borrower(reinterpret_cast(arena) | Owner::kArenaBit); - } - - static Borrower Arena(std::nullptr_t) = delete; - - static Borrower ReferenceCount( - const ReferenceCount* absl_nonnull reference_count - ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ABSL_DCHECK(reference_count != nullptr); - return Borrower(reinterpret_cast(reference_count) | - Owner::kReferenceCountBit); - } - - static Borrower ReferenceCount(std::nullptr_t) = delete; - - Borrower() = default; - Borrower(const Borrower&) = default; - Borrower(Borrower&&) = default; - Borrower& operator=(const Borrower&) = default; - Borrower& operator=(Borrower&&) = default; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower(const Owned& owned ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower(Borrowed borrowed) noexcept; - - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower(const Owner& owner ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept - : ptr_(owner.ptr_) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower& operator=( - const Owner& owner ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - ptr_ = owner.ptr_; - return *this; - } - - Borrower& operator=(Owner&&) = delete; - - template - Borrower& operator=( - const Owned& owned ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept; - - template - Borrower& operator=(Owned&&) = delete; - - template - // NOLINTNEXTLINE(google-explicit-constructor) - Borrower& operator=(Borrowed borrowed) noexcept; - - explicit operator bool() const noexcept { return !Owner::IsNone(ptr_); } - - google::protobuf::Arena* absl_nullable arena() const noexcept { - return (ptr_ & Owner::kBits) == Owner::kArenaBit - ? reinterpret_cast(ptr_ & Owner::kPointerMask) - : nullptr; - } - - void reset() noexcept { ptr_ = 0; } - - // Tests whether two borrowers are borrowing the same data. - friend bool operator==(Borrower lhs, Borrower rhs) noexcept { - // A reference count and arena can never occupy the same memory address, so - // we can compare for equality without masking off the bits. - return lhs.ptr_ == rhs.ptr_; - } - - private: - friend class Owner; - template - friend struct Borrowable; - friend const common_internal::ReferenceCount* absl_nullable - common_internal::BorrowerRelease(Borrower borrower) noexcept; - - constexpr explicit Borrower(uintptr_t ptr) noexcept : ptr_(ptr) {} - - uintptr_t ptr_ = Owner::kNone; -}; - -inline bool operator!=(Borrower lhs, Borrower rhs) noexcept { - return !operator==(lhs, rhs); -} - -inline bool operator==(Borrower lhs, const Owner& rhs) noexcept { - return operator==(lhs, Borrower(rhs)); -} - -inline bool operator==(const Owner& lhs, Borrower rhs) noexcept { - return operator==(Borrower(lhs), rhs); -} - -inline bool operator!=(Borrower lhs, const Owner& rhs) noexcept { - return !operator==(lhs, rhs); -} - -inline bool operator!=(const Owner& lhs, Borrower rhs) noexcept { - return !operator==(lhs, rhs); -} - -inline Owner::Owner(Borrower borrower) noexcept - : ptr_(Owner::Own(borrower.ptr_)) {} - -namespace common_internal { - -inline const ReferenceCount* absl_nullable BorrowerRelease( - Borrower borrower) noexcept { - uintptr_t ptr = borrower.ptr_; - if (Owner::IsReferenceCount(ptr)) { - return Owner::AsReferenceCount(ptr); - } - return nullptr; -} - -} // namespace common_internal - -template -Unique AllocateUnique(Allocator<> allocator, Args&&... args); - -// Wrap an already created `T` in `Unique`. Requires that `T` is not const, -// otherwise `GetArena()` may return slightly unexpected results depending on if -// it is the default value. -template -std::enable_if_t, Unique> WrapUnique(T* object); - -template -Unique WrapUnique(T* object, Allocator<> allocator); - -// `Unique` points to an object which was allocated using `Allocator<>` or -// `Allocator`. It has ownership over the object, and will perform any -// destruction and deallocation required. `Unique` must not outlive the -// underlying arena, if any. Unlike `Owned` and `Borrowed`, `Unique` supports -// arena incompatible objects. It is very similar to `std::unique_ptr` when -// using a custom deleter. -// -// IMPLEMENTATION NOTES: -// When utilizing arenas, we optionally perform a risky optimization via -// `AllocateUnique`. We do not use `Arena::Create`, instead we directly allocate -// the bytes and construct it in place ourselves. This avoids registering the -// destructor when required. Instead we register the destructor ourselves, if -// required, during `Unique::release`. This allows us to avoid deferring -// destruction of the object until the arena is destroyed, avoiding the cost -// involved in doing so. -template -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Unique final { - public: - using element_type = T; - - static_assert(!std::is_array_v, "T must not be an array"); - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_volatile_v, "T must not be volatile qualified"); - - Unique() = default; - Unique(const Unique&) = delete; - Unique& operator=(const Unique&) = delete; - - explicit Unique(T* ptr) noexcept - : Unique(ptr, common_internal::GetArena(ptr)) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - Unique(std::nullptr_t) noexcept : Unique() {} - - Unique(Unique&& other) noexcept : Unique(other.ptr_, other.arena_) { - other.ptr_ = nullptr; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Unique(Unique&& other) noexcept : Unique(other.ptr_, other.arena_) { - other.ptr_ = nullptr; - } - - ~Unique() { Delete(); } - - Unique& operator=(Unique&& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - Delete(); - ptr_ = other.ptr_; - arena_ = other.arena_; - other.ptr_ = nullptr; - } - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Unique& operator=(U* other) noexcept { - reset(other); - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Unique& operator=(Unique&& other) noexcept { - Delete(); - ptr_ = other.ptr_; - arena_ = other.arena_; - other.ptr_ = nullptr; - return *this; - } - - // NOLINTNEXTLINE(google-explicit-constructor) - Unique& operator=(std::nullptr_t) noexcept { - reset(); - return *this; - } - - T& operator*() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(static_cast(*this)); - return *get(); - } - - T* absl_nonnull operator->() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(static_cast(*this)); - return get(); - } - - // Relinquishes ownership of `T*`, returning it. If `T` was allocated and - // constructed using an arena, no further action is required. If `T` was - // allocated and constructed without an arena, the caller must eventually call - // `delete`. - ABSL_MUST_USE_RESULT T* release() noexcept { - PreRelease(); - return std::exchange(ptr_, nullptr); - } - - void reset() noexcept { reset(nullptr); } - - void reset(T* ptr) noexcept { - Delete(); - ptr_ = ptr; - arena_ = reinterpret_cast(common_internal::GetArena(ptr)); - } - - void reset(std::nullptr_t) noexcept { - Delete(); - ptr_ = nullptr; - arena_ = 0; - } - - explicit operator bool() const noexcept { return get() != nullptr; } - - google::protobuf::Arena* absl_nullable arena() const noexcept { - return reinterpret_cast( - arena_ & common_internal::kUniqueArenaPointerMask); - } - - friend void swap(Unique& lhs, Unique& rhs) noexcept { - using std::swap; - swap(lhs.ptr_, rhs.ptr_); - swap(lhs.arena_, rhs.arena_); - } - - private: - template - friend class Unique; - template - friend class Owned; - template - friend Unique AllocateUnique(Allocator<> allocator, Args&&... args); - template - friend Unique WrapUnique(U* object, Allocator<> allocator); - friend class ReferenceCountingMemoryManager; - friend class PoolingMemoryManager; - friend struct std::pointer_traits>; - friend struct ArenaTraits>; - - static constexpr bool kNeedsArenaDestructor = - !std::is_trivially_destructible_v && - !google::protobuf::Arena::is_destructor_skippable::value && - !std::is_base_of_v; - - Unique(T* ptr, uintptr_t arena) noexcept : ptr_(ptr), arena_(arena) {} - - Unique(T* ptr, google::protobuf::Arena* arena, bool unowned = false) noexcept - : Unique(ptr, - reinterpret_cast(arena) | - (unowned ? common_internal::kUniqueArenaUnownedBit : 0)) { - ABSL_ASSERT(!unowned || (unowned && arena != nullptr)); - } - - Unique(google::protobuf::Arena* arena, T* ptr, bool unowned = false) noexcept - : Unique(ptr, arena, unowned) {} - - T* get() const noexcept { return ptr_; } - - void Delete() const noexcept { - if (static_cast(*this)) { - if (arena_ != 0) { - if ((arena_ & common_internal::kUniqueArenaBits) == - common_internal::kUniqueArenaUnownedBit) { - // We never registered the destructor, call it if necessary. - if constexpr (kNeedsArenaDestructor) { - std::destroy_at(ptr_); - } - } - } else { - delete ptr_; - } - } - } - - void PreRelease() noexcept { - if constexpr (kNeedsArenaDestructor) { - if (static_cast(*this) && - (arena_ & common_internal::kUniqueArenaBits) == - common_internal::kUniqueArenaUnownedBit) { - // We never registered the destructor, call it if necessary. - arena()->OwnDestructor(const_cast*>(ptr_)); - arena_ &= common_internal::kUniqueArenaPointerMask; - } - } - } - - void Release(T** ptr, Owner* owner) noexcept { - if (ptr_ == nullptr) { - *ptr = nullptr; - return; - } - PreRelease(); - *ptr = std::exchange(ptr_, nullptr); - if (arena_ == 0) { - owner->ptr_ = reinterpret_cast( - common_internal::MakeDeletingReferenceCount(*ptr)) | - common_internal::kMetadataOwnerReferenceCountBit; - } else { - owner->ptr_ = reinterpret_cast(arena()) | - common_internal::kMetadataOwnerArenaBit; - } - } - - T* ptr_ = nullptr; - // Potentially tagged pointer to `google::protobuf::Arena`. The tag is used to determine - // whether we still need to register the destructor with the `google::protobuf::Arena`. - uintptr_t arena_ = 0; -}; - -template -Unique(T*) -> Unique; - -template -Unique AllocateUnique(Allocator<> allocator, Args&&... args) { - using U = std::remove_cv_t; - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_array_v, "T must not be an array"); - - U* object; - google::protobuf::Arena* absl_nullable arena = allocator.arena(); - bool unowned; - if constexpr (google::protobuf::Arena::is_arena_constructable::value) { - object = google::protobuf::Arena::Create(arena, std::forward(args)...); - // For arena-compatible proto types, let the Arena::Create handle - // registering the destructor call. - // Otherwise, Unique retains a pointer to the owning arena so it may - // conditionally register T::~T depending on usage. - unowned = false; - } else { - void* p = allocator.allocate_bytes(sizeof(U), alignof(U)); - CEL_INTERNAL_TRY { - if constexpr (ArenaTraits<>::constructible()) { - object = ::new (p) U(arena, std::forward(args)...); - } else { - object = ::new (p) U(std::forward(args)...); - } - } - CEL_INTERNAL_CATCH_ANY { - allocator.deallocate_bytes(p, sizeof(U), alignof(U)); - CEL_INTERNAL_RETHROW; - } - unowned = - arena != nullptr && !ArenaTraits<>::trivially_destructible(*object); - } - return Unique(object, arena, unowned); -} - -template -std::enable_if_t, Unique> WrapUnique(T* object) { - return Unique(object); -} - -template -Unique WrapUnique(T* object, Allocator<> allocator) { - return Unique(object, allocator.arena()); -} - -template -inline bool operator==(const Unique& lhs, std::nullptr_t) { - return !static_cast(lhs); -} - -template -inline bool operator==(std::nullptr_t, const Unique& rhs) { - return !static_cast(rhs); -} - -template -inline bool operator!=(const Unique& lhs, std::nullptr_t) { - return static_cast(lhs); -} - -template -inline bool operator!=(std::nullptr_t, const Unique& rhs) { - return static_cast(rhs); -} - -} // namespace cel - -namespace std { - -template -struct pointer_traits> { - using pointer = cel::Unique; - using element_type = typename cel::Unique::element_type; - using difference_type = ptrdiff_t; - - template - using rebind = cel::Unique; - - static element_type* to_address(const pointer& p) noexcept { return p.ptr_; } -}; - -} // namespace std - -namespace cel { - -template -struct ArenaTraits> { - static bool trivially_destructible(const Unique& unique) { - return unique.arena_ != 0 && - (unique.arena_ & common_internal::kUniqueArenaBits) == 0; - } -}; - -// `Owned` points to an object which was allocated using `Allocator<>` or -// `Allocator`. It has co-ownership over the object. `T` must meet the named -// requirement `ArenaConstructable`. -template -class ABSL_ATTRIBUTE_TRIVIAL_ABI [[nodiscard]] Owned final { - public: - using element_type = T; - - static_assert(!std::is_array_v, "T must not be an array"); - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_volatile_v, "T must not be volatile qualified"); - static_assert(!std::is_void_v, "T must not be void"); - - Owned() = default; - Owned(const Owned&) = default; - Owned& operator=(const Owned&) = default; - - Owned(Owned&& other) noexcept - : Owned(std::exchange(other.value_, nullptr), std::move(other.owner_)) {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned(const Owned& other) noexcept : Owned(other.value_, other.owner_) {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned(Owned&& other) noexcept - : Owned(std::exchange(other.value_, nullptr), std::move(other.owner_)) {} - - template >> - explicit Owned(Borrowed other) noexcept; - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned(Unique&& other) : Owned() { - other.Release(&value_, &owner_); - } - - Owned(Owner owner, T* value ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept - : Owned(value, std::move(owner)) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - Owned(std::nullptr_t) noexcept : Owned() {} - - Owned& operator=(Owned&& other) noexcept { - if (ABSL_PREDICT_TRUE(this != &other)) { - value_ = std::exchange(other.value_, nullptr); - owner_ = std::move(other.owner_); - } - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(const Owned& other) noexcept { - value_ = other.value_; - owner_ = other.owner_; - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(Owned&& other) noexcept { - value_ = std::exchange(other.value_, nullptr); - owner_ = std::move(other.owner_); - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(Borrowed other) noexcept; - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(Unique&& other) { - owner_.reset(); - other.Release(&value_, &owner_); - return *this; - } - - // NOLINTNEXTLINE(google-explicit-constructor) - Owned& operator=(std::nullptr_t) noexcept { - reset(); - return *this; - } - - T& operator*() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(static_cast(*this)); - return *get(); - } - - T* absl_nonnull operator->() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(static_cast(*this)); - return get(); - } - - void reset() noexcept { - value_ = nullptr; - owner_.reset(); - } - - google::protobuf::Arena* absl_nullable arena() const noexcept { return owner_.arena(); } - - explicit operator bool() const noexcept { return get() != nullptr; } - - friend void swap(Owned& lhs, Owned& rhs) noexcept { - using std::swap; - swap(lhs.value_, rhs.value_); - swap(lhs.owner_, rhs.owner_); - } - - private: - friend class Owner; - friend class Borrower; - template - friend class Owned; - template - friend class Borrowed; - template - friend struct Ownable; - template - friend Owned AllocateShared(Allocator<> allocator, Args&&... args); - template - friend Owned WrapShared(U* object, Allocator<> allocator); - template - friend Owned common_internal::WrapEternal(const U* value); - friend struct std::pointer_traits>; - friend struct ArenaTraits>; - - Owned(T* value, Owner owner) noexcept - : value_(value), owner_(std::move(owner)) {} - - T* get() const noexcept { return value_; } - - T* value_ = nullptr; - Owner owner_; -}; - -template -Owned(T*) -> Owned; -template -Owned(Unique) -> Owned; -template -Owned(Owner, T*) -> Owned; -template -Owned(Borrowed) -> Owned; - -} // namespace cel - -namespace std { - -template -struct pointer_traits> { - using pointer = cel::Owned; - using element_type = typename cel::Owned::element_type; - using difference_type = ptrdiff_t; - - template - using rebind = cel::Owned; - - static element_type* to_address(const pointer& p) noexcept { - return p.value_; - } -}; - -} // namespace std - -namespace cel { - -template -struct ArenaTraits> { - static bool trivially_destructible(const Owned& owned) { - return ArenaTraits<>::trivially_destructible(owned.owner_); - } -}; - -template -Owner::Owner(const Owned& owned) noexcept : Owner(owned.owner_) {} - -template -Owner::Owner(Owned&& owned) noexcept : Owner(std::move(owned.owner_)) { - owned.value_ = nullptr; -} - -template -Owner& Owner::operator=(const Owned& owned) noexcept { - *this = owned.owner_; - return *this; -} - -template -Owner& Owner::operator=(Owned&& owned) noexcept { - *this = std::move(owned.owner_); - owned.value_ = nullptr; - return *this; -} - -template -bool operator==(const Owned& lhs, std::nullptr_t) noexcept { - return !static_cast(lhs); -} - -template -bool operator==(std::nullptr_t, const Owned& rhs) noexcept { - return rhs == nullptr; -} - -template -bool operator!=(const Owned& lhs, std::nullptr_t) noexcept { - return !operator==(lhs, nullptr); -} - -template -bool operator!=(std::nullptr_t, const Owned& rhs) noexcept { - return !operator==(nullptr, rhs); -} - -template -Owned AllocateShared(Allocator<> allocator, Args&&... args) { - using U = std::remove_cv_t; - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_array_v, "T must not be an array"); - - U* object; - Owner owner; - if (google::protobuf::Arena* absl_nullable arena = allocator.arena(); - arena != nullptr) { - object = ArenaAllocator(arena).template new_object( - std::forward(args)...); - owner.ptr_ = reinterpret_cast(arena) | - common_internal::kMetadataOwnerArenaBit; - } else { - const common_internal::ReferenceCount* refcount; - std::tie(object, refcount) = common_internal::MakeEmplacedReferenceCount( - std::forward(args)...); - owner.ptr_ = reinterpret_cast(refcount) | - common_internal::kMetadataOwnerReferenceCountBit; - } - return Owned(object, std::move(owner)); -} - -template -Owned WrapShared(T* object, Allocator<> allocator) { - Owner owner; - if (object == nullptr) { - } else if (allocator.arena() != nullptr) { - owner.ptr_ = reinterpret_cast( - static_cast(allocator.arena())) | - common_internal::kMetadataOwnerArenaBit; - } else { - owner.ptr_ = reinterpret_cast( - common_internal::MakeDeletingReferenceCount(object)) | - common_internal::kMetadataOwnerReferenceCountBit; - } - return Owned(object, std::move(owner)); -} - -template -std::enable_if_t, Owned> WrapShared(T* object) { - return WrapShared(object, object->GetArena()); -} - -namespace common_internal { - -template -Owned WrapEternal(const T* value) { - return Owned(value, Owner::None()); -} - -} // namespace common_internal - -// `Borrowed` points to an object which was allocated using `Allocator<>` or -// `Allocator`. It has no ownership over the object, and is only valid so -// long as one or more owners of the object exist. `T` must meet the named -// requirement `ArenaConstructable`. -template -class Borrowed final { - public: - using element_type = T; - - static_assert(!std::is_array_v, "T must not be an array"); - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_volatile_v, "T must not be volatile qualified"); - static_assert(!std::is_void_v, "T must not be void"); - - Borrowed() = default; - Borrowed(const Borrowed&) = default; - Borrowed(Borrowed&&) = default; - Borrowed& operator=(const Borrowed&) = default; - Borrowed& operator=(Borrowed&&) = default; - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed(const Borrowed& other) noexcept - : Borrowed(other.value_, other.borrower_) {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed(Borrowed&& other) noexcept - : Borrowed(other.value_, other.borrower_) {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed(const Owned& other ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept - : Borrowed(other.value_, other.owner_) {} - - Borrowed(Borrower borrower, T* ptr) noexcept : Borrowed(ptr, borrower) {} - - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed(std::nullptr_t) noexcept : Borrowed() {} - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=(const Borrowed& other) noexcept { - value_ = other.value_; - borrower_ = other.borrower_; - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=(Borrowed&& other) noexcept { - value_ = other.value_; - borrower_ = other.borrower_; - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=( - const Owned& other ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - value_ = other.value_; - borrower_ = other.borrower_; - return *this; - } - - template >> - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=(Owned&&) = delete; - - // NOLINTNEXTLINE(google-explicit-constructor) - Borrowed& operator=(std::nullptr_t) noexcept { - reset(); - return *this; - } - - T& operator*() const noexcept { - ABSL_DCHECK(static_cast(*this)); - return *get(); - } - - T* absl_nonnull operator->() const noexcept { - ABSL_DCHECK(static_cast(*this)); - return get(); - } - - void reset() noexcept { - value_ = nullptr; - borrower_.reset(); - } - - google::protobuf::Arena* absl_nullable arena() const noexcept { - return borrower_.arena(); - } - - explicit operator bool() const noexcept { return get() != nullptr; } - - private: - friend class Owner; - friend class Borrower; - template - friend class Owned; - template - friend class Borrowed; - template - friend struct Borrowable; - friend struct std::pointer_traits>; - - constexpr Borrowed(T* value, Borrower borrower) noexcept - : value_(value), borrower_(borrower) {} - - T* get() const noexcept { return value_; } - - T* value_ = nullptr; - Borrower borrower_; -}; - -template -Borrowed(T*) -> Borrowed; -template -Borrowed(Borrower, T*) -> Borrowed; -template -Borrowed(Owned) -> Borrowed; - -} // namespace cel - -namespace std { - -template -struct pointer_traits> { - using pointer = cel::Borrowed; - using element_type = typename cel::Borrowed::element_type; - using difference_type = ptrdiff_t; - - template - using rebind = cel::Borrowed; - - static element_type* to_address(pointer p) noexcept { return p.value_; } -}; - -} // namespace std - -namespace cel { - -template -Owner::Owner(Borrowed borrowed) noexcept : Owner(borrowed.borrower_) {} - -template -Borrower::Borrower(const Owned& owned ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept - : Borrower(owned.owner_) {} - -template -Borrower::Borrower(Borrowed borrowed) noexcept - : Borrower(borrowed.borrower_) {} - -template -Borrower& Borrower::operator=( - const Owned& owned ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept { - *this = owned.owner_; - return *this; -} - -template -Borrower& Borrower::operator=(Borrowed borrowed) noexcept { - *this = borrowed.borrower_; - return *this; -} - -template -bool operator==(Borrowed lhs, std::nullptr_t) noexcept { - return !static_cast(lhs); -} - -template -bool operator==(std::nullptr_t, Borrowed rhs) noexcept { - return rhs == nullptr; -} - -template -bool operator!=(Borrowed lhs, std::nullptr_t) noexcept { - return !operator==(lhs, nullptr); -} - -template -bool operator!=(std::nullptr_t, Borrowed rhs) noexcept { - return !operator==(nullptr, rhs); -} - -template -template -Owned::Owned(Borrowed other) noexcept - : Owned(other.value_, Owner(other.borrower_)) {} - -template -template -Owned& Owned::operator=(Borrowed other) noexcept { - value_ = other.value_; - owner_ = Owner(other.borrower_); - return *this; -} - -// `Ownable` is a mixin for enabling the ability to get `Owned` that refer to -// this. -template -struct Ownable { - protected: - Owned Own() const noexcept { - static_assert(std::is_base_of_v, "T must be derived from Data"); - const T* const that = static_cast(this); - return Owned( - Owner(Owner::Own(static_cast(that)->owner_)), that); - } - - Owned Own() noexcept { - static_assert(std::is_base_of_v, "T must be derived from Data"); - T* const that = static_cast(this); - return Owned(Owner(Owner::Own(static_cast(that)->owner_)), that); - } - - ABSL_DEPRECATED("Use Own") - Owned shared_from_this() const noexcept { return Own(); } - - ABSL_DEPRECATED("Use Own") - Owned shared_from_this() noexcept { return Own(); } -}; - -// `Borrowable` is a mixin for enabling the ability to get `Borrowed` that -// refer to this. -template -struct Borrowable { - protected: - Borrowed Borrow() const noexcept { - static_assert(std::is_base_of_v, "T must be derived from Data"); - const T* const that = static_cast(this); - return Borrowed(Borrower(static_cast(that)->owner_), - that); - } - - Borrowed Borrow() noexcept { - static_assert(std::is_base_of_v, "T must be derived from Data"); - T* const that = static_cast(this); - return Borrowed(Borrower(static_cast(that)->owner_), that); - } -}; - // `ReferenceCountingMemoryManager` is a `MemoryManager` which employs automatic // memory management through reference counting. class ReferenceCountingMemoryManager final { diff --git a/common/memory_test.cc b/common/memory_test.cc deleted file mode 100644 index 7f3e7a82a..000000000 --- a/common/memory_test.cc +++ /dev/null @@ -1,466 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This header contains primitives for reference counting, roughly equivalent to -// the primitives used to implement `std::shared_ptr`. These primitives should -// not be used directly in most cases, instead `cel::ManagedMemory` should be -// used instead. - -#include "common/memory.h" - -#include - -#include "google/protobuf/struct.pb.h" -#include "absl/base/nullability.h" -#include "common/allocator.h" -#include "common/data.h" -#include "common/internal/reference_count.h" -#include "internal/testing.h" -#include "google/protobuf/arena.h" - -#ifdef ABSL_HAVE_EXCEPTIONS -#include -#endif - -namespace cel { -namespace { - -using ::testing::IsFalse; -using ::testing::IsNull; -using ::testing::IsTrue; -using ::testing::TestParamInfo; -using ::testing::TestWithParam; - -TEST(Owner, None) { - EXPECT_THAT(Owner::None(), IsFalse()); - EXPECT_THAT(Owner::None().arena(), IsNull()); -} - -TEST(Owner, Allocator) { - google::protobuf::Arena arena; - EXPECT_THAT(Owner::Allocator(NewDeleteAllocator<>{}), IsFalse()); - EXPECT_THAT(Owner::Allocator(ArenaAllocator<>{&arena}), IsTrue()); -} - -TEST(Owner, Arena) { - google::protobuf::Arena arena; - EXPECT_THAT(Owner::Arena(&arena), IsTrue()); - EXPECT_EQ(Owner::Arena(&arena).arena(), &arena); -} - -TEST(Owner, ReferenceCount) { - auto* refcount = new common_internal::ReferenceCounted(); - EXPECT_THAT(Owner::ReferenceCount(refcount), IsTrue()); - EXPECT_THAT(Owner::ReferenceCount(refcount).arena(), IsNull()); - common_internal::StrongUnref(refcount); -} - -TEST(Owner, Equality) { - google::protobuf::Arena arena1; - google::protobuf::Arena arena2; - EXPECT_EQ(Owner::None(), Owner::None()); - EXPECT_EQ(Owner::Allocator(NewDeleteAllocator<>{}), Owner::None()); - EXPECT_EQ(Owner::Arena(&arena1), Owner::Arena(&arena1)); - EXPECT_NE(Owner::Arena(&arena1), Owner::None()); - EXPECT_NE(Owner::None(), Owner::Arena(&arena1)); - EXPECT_NE(Owner::Arena(&arena1), Owner::Arena(&arena2)); - EXPECT_EQ(Owner::Allocator(ArenaAllocator<>{&arena1}), Owner::Arena(&arena1)); -} - -TEST(Borrower, None) { - EXPECT_THAT(Borrower::None(), IsFalse()); - EXPECT_THAT(Borrower::None().arena(), IsNull()); -} - -TEST(Borrower, Allocator) { - google::protobuf::Arena arena; - EXPECT_THAT(Borrower::Allocator(NewDeleteAllocator<>{}), IsFalse()); - EXPECT_THAT(Borrower::Allocator(ArenaAllocator<>{&arena}), IsTrue()); -} - -TEST(Borrower, Arena) { - google::protobuf::Arena arena; - EXPECT_THAT(Borrower::Arena(&arena), IsTrue()); - EXPECT_EQ(Borrower::Arena(&arena).arena(), &arena); -} - -TEST(Borrower, ReferenceCount) { - auto* refcount = new common_internal::ReferenceCounted(); - EXPECT_THAT(Borrower::ReferenceCount(refcount), IsTrue()); - EXPECT_THAT(Borrower::ReferenceCount(refcount).arena(), IsNull()); - common_internal::StrongUnref(refcount); -} - -TEST(Borrower, Equality) { - google::protobuf::Arena arena1; - google::protobuf::Arena arena2; - EXPECT_EQ(Borrower::None(), Borrower::None()); - EXPECT_EQ(Borrower::Allocator(NewDeleteAllocator<>{}), Borrower::None()); - EXPECT_EQ(Borrower::Arena(&arena1), Borrower::Arena(&arena1)); - EXPECT_NE(Borrower::Arena(&arena1), Borrower::None()); - EXPECT_NE(Borrower::None(), Borrower::Arena(&arena1)); - EXPECT_NE(Borrower::Arena(&arena1), Borrower::Arena(&arena2)); - EXPECT_EQ(Borrower::Allocator(ArenaAllocator<>{&arena1}), - Borrower::Arena(&arena1)); -} - -TEST(OwnerBorrower, CopyConstruct) { - auto* refcount = new common_internal::ReferenceCounted(); - Owner owner1 = Owner::ReferenceCount(refcount); - common_internal::StrongUnref(refcount); - Owner owner2(owner1); - Borrower borrower(owner1); - EXPECT_EQ(owner1, owner2); - EXPECT_EQ(owner1, borrower); - EXPECT_EQ(borrower, owner1); -} - -TEST(OwnerBorrower, MoveConstruct) { - auto* refcount = new common_internal::ReferenceCounted(); - Owner owner1 = Owner::ReferenceCount(refcount); - common_internal::StrongUnref(refcount); - Owner owner2(std::move(owner1)); - Borrower borrower(owner2); - EXPECT_EQ(owner2, borrower); - EXPECT_EQ(borrower, owner2); -} - -TEST(OwnerBorrower, CopyAssign) { - auto* refcount = new common_internal::ReferenceCounted(); - Owner owner1 = Owner::ReferenceCount(refcount); - common_internal::StrongUnref(refcount); - Owner owner2; - owner2 = owner1; - Borrower borrower(owner1); - EXPECT_EQ(owner1, owner2); - EXPECT_EQ(owner1, borrower); - EXPECT_EQ(borrower, owner1); -} - -TEST(OwnerBorrower, MoveAssign) { - auto* refcount = new common_internal::ReferenceCounted(); - Owner owner1 = Owner::ReferenceCount(refcount); - common_internal::StrongUnref(refcount); - Owner owner2; - owner2 = std::move(owner1); - Borrower borrower(owner2); - EXPECT_EQ(owner2, borrower); - EXPECT_EQ(borrower, owner2); -} - -TEST(Unique, ToAddress) { - Unique unique; - EXPECT_EQ(cel::to_address(unique), nullptr); - unique = AllocateUnique(NewDeleteAllocator<>{}); - EXPECT_EQ(cel::to_address(unique), unique.operator->()); -} - -class OwnedTest : public TestWithParam { - public: - Allocator<> GetAllocator() { - switch (GetParam()) { - case AllocatorKind::kArena: - return ArenaAllocator<>{&arena_}; - case AllocatorKind::kNewDelete: - return NewDeleteAllocator<>{}; - } - } - - private: - google::protobuf::Arena arena_; -}; - -TEST_P(OwnedTest, Default) { - Owned owned; - EXPECT_FALSE(owned); - EXPECT_EQ(cel::to_address(owned), nullptr); - EXPECT_FALSE(owned != nullptr); - EXPECT_FALSE(nullptr != owned); -} - -class TestData final : public Data { - public: - using InternalArenaConstructable_ = void; - using DestructorSkippable_ = void; - - TestData() noexcept : Data() {} - - explicit TestData(google::protobuf::Arena* absl_nullable arena) noexcept - : Data(arena) {} -}; - -TEST_P(OwnedTest, AllocateSharedData) { - auto owned = AllocateShared(GetAllocator()); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, AllocateSharedMessageLite) { - auto owned = AllocateShared(GetAllocator()); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, WrapSharedData) { - auto owned = - WrapShared(google::protobuf::Arena::Create(GetAllocator().arena())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, WrapSharedMessageLite) { - auto owned = WrapShared( - google::protobuf::Arena::Create(GetAllocator().arena())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, SharedFromUniqueData) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, SharedFromUniqueMessageLite) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_EQ(Owner(owned).arena(), GetAllocator().arena()); - EXPECT_EQ(Borrower(owned).arena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, CopyConstruct) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned copied_owned(owned); - EXPECT_EQ(copied_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, MoveConstruct) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned moved_owned(std::move(owned)); - EXPECT_EQ(moved_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, CopyConstructOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned copied_owned(owned); - EXPECT_EQ(copied_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, MoveConstructOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned moved_owned(std::move(owned)); - EXPECT_EQ(moved_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, ConstructBorrowed) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned borrowed_owned(Borrowed{owned}); - EXPECT_EQ(borrowed_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, ConstructOwner) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned owner_owned(Owner(owned), cel::to_address(owned)); - EXPECT_EQ(owner_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, ConstructNullPtr) { - Owned owned(nullptr); - EXPECT_EQ(owned, nullptr); -} - -TEST_P(OwnedTest, CopyAssign) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned copied_owned; - copied_owned = owned; - EXPECT_EQ(copied_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, MoveAssign) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned moved_owned; - moved_owned = std::move(owned); - EXPECT_EQ(moved_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, CopyAssignOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned copied_owned; - copied_owned = owned; - EXPECT_EQ(copied_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, MoveAssignOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned moved_owned; - moved_owned = std::move(owned); - EXPECT_EQ(moved_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, AssignBorrowed) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Owned borrowed_owned; - borrowed_owned = Borrowed{owned}; - EXPECT_EQ(borrowed_owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, AssignUnique) { - Owned owned; - owned = AllocateUnique(GetAllocator()); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); -} - -TEST_P(OwnedTest, AssignNullPtr) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - EXPECT_TRUE(owned); - owned = nullptr; - EXPECT_FALSE(owned); -} - -INSTANTIATE_TEST_SUITE_P(OwnedTest, OwnedTest, - ::testing::Values(AllocatorKind::kArena, - AllocatorKind::kNewDelete)); - -class BorrowedTest : public TestWithParam { - public: - Allocator<> GetAllocator() { - switch (GetParam()) { - case AllocatorKind::kArena: - return ArenaAllocator<>{&arena_}; - case AllocatorKind::kNewDelete: - return NewDeleteAllocator<>{}; - } - } - - private: - google::protobuf::Arena arena_; -}; - -TEST_P(BorrowedTest, Default) { - Borrowed borrowed; - EXPECT_FALSE(borrowed); - EXPECT_EQ(cel::to_address(borrowed), nullptr); - EXPECT_FALSE(borrowed != nullptr); - EXPECT_FALSE(nullptr != borrowed); -} - -TEST_P(BorrowedTest, CopyConstruct) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed copied_borrowed(borrowed); - EXPECT_EQ(copied_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, MoveConstruct) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed moved_borrowed(std::move(borrowed)); - EXPECT_EQ(moved_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, CopyConstructOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed copied_borrowed(borrowed); - EXPECT_EQ(copied_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, MoveConstructOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed moved_borrowed(std::move(borrowed)); - EXPECT_EQ(moved_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, ConstructNullPtr) { - Borrowed borrowed(nullptr); - EXPECT_FALSE(borrowed); -} - -TEST_P(BorrowedTest, CopyAssign) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed copied_borrowed; - copied_borrowed = borrowed; - EXPECT_EQ(copied_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, MoveAssign) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed moved_borrowed; - moved_borrowed = std::move(borrowed); - EXPECT_EQ(moved_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, CopyAssignOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed copied_borrowed; - copied_borrowed = borrowed; - EXPECT_EQ(copied_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, MoveAssignOther) { - auto owned = Owned(AllocateUnique(GetAllocator())); - auto borrowed = Borrowed(owned); - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); - Borrowed moved_borrowed; - moved_borrowed = std::move(borrowed); - EXPECT_EQ(moved_borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, AssignOwned) { - auto owned = Owned(AllocateUnique(GetAllocator())); - EXPECT_EQ(owned->GetArena(), GetAllocator().arena()); - Borrowed borrowed = owned; - EXPECT_EQ(borrowed->GetArena(), GetAllocator().arena()); -} - -TEST_P(BorrowedTest, AssignNullPtr) { - Borrowed borrowed; - borrowed = nullptr; - EXPECT_FALSE(borrowed); -} - -INSTANTIATE_TEST_SUITE_P(BorrowedTest, BorrowedTest, - ::testing::Values(AllocatorKind::kArena, - AllocatorKind::kNewDelete)); - -} // namespace -} // namespace cel diff --git a/common/value.cc b/common/value.cc index cc3a0825f..fa937bdf0 100644 --- a/common/value.cc +++ b/common/value.cc @@ -43,7 +43,6 @@ #include "absl/types/source_location.h" #include "absl/types/variant.h" #include "common/allocator.h" -#include "common/memory.h" #include "common/optional_ref.h" #include "common/type.h" #include "common/value_kind.h" @@ -1244,7 +1243,8 @@ struct OwningWellKnownTypesValueVisitor { cloned->CopyFrom(*value); return ParsedJsonListValue(cloned, arena); } - return ParsedJsonListValue(value.release(), arena); + return ParsedJsonListValue(value.try_as_arena_ptr()->get(), + arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1258,23 +1258,23 @@ struct OwningWellKnownTypesValueVisitor { return ParsedJsonMapValue(cloned, arena); }, [&](well_known_types::StructPtr value) -> MapValue { - if (value.arena() != arena) { + if (value.GetOwningArena() != arena) { auto* cloned = value->New(arena); cloned->CopyFrom(*value); return ParsedJsonMapValue(cloned, arena); } - return ParsedJsonMapValue(value.release(), arena); + return ParsedJsonMapValue(value.try_as_arena_ptr()->get(), arena); }), well_known_types::AsVariant(std::move(value))); } - Value operator()(Unique value) const { + Value operator()(google::protobuf::Arena::UniquePtr value) const { if (value->GetArena() != arena) { auto* cloned = value->New(arena); cloned->CopyFrom(*value); return ParsedMessageValue(cloned, arena); } - return ParsedMessageValue(value.release(), arena); + return ParsedMessageValue(value.try_as_arena_ptr()->get(), arena); } template @@ -1344,7 +1344,8 @@ struct BorrowingWellKnownTypesValueVisitor { cloned->CopyFrom(*value); return ParsedJsonListValue(cloned, arena); } - return ParsedJsonListValue(value.release(), arena); + return ParsedJsonListValue(value.try_as_arena_ptr()->get(), + arena); }), well_known_types::AsVariant(std::move(value))); } @@ -1362,18 +1363,18 @@ struct BorrowingWellKnownTypesValueVisitor { cloned->CopyFrom(*value); return ParsedJsonMapValue(cloned, arena); } - return ParsedJsonMapValue(value.release(), arena); + return ParsedJsonMapValue(value.try_as_arena_ptr()->get(), arena); }), well_known_types::AsVariant(std::move(value))); } - Value operator()(Unique&& value) const { + Value operator()(google::protobuf::Arena::UniquePtr&& value) const { if (value->GetArena() != arena) { auto* cloned = value->New(arena); cloned->CopyFrom(*value); return ParsedMessageValue(cloned, arena); } - return ParsedMessageValue(value.release(), arena); + return ParsedMessageValue(value.try_as_arena_ptr()->get(), arena); } template diff --git a/internal/BUILD b/internal/BUILD index d6781b538..e49dd67e3 100644 --- a/internal/BUILD +++ b/internal/BUILD @@ -727,7 +727,6 @@ cc_test( ":testing_descriptor_pool", ":testing_message_factory", ":well_known_types", - "//common:memory", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:die_if_null", @@ -737,7 +736,6 @@ cc_test( "@com_google_absl//absl/strings:cord", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/time", - "@com_google_absl//absl/types:variant", "@com_google_cel_spec//proto/cel/expr/conformance/proto3:test_all_types_cc_proto", "@com_google_protobuf//:any_cc_proto", "@com_google_protobuf//:duration_cc_proto", @@ -842,12 +840,12 @@ cc_test( ":testing_descriptor_pool", ":testing_message_factory", ":well_known_types", - "//common:allocator", "//common:memory", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:die_if_null", + "@com_google_absl//absl/memory", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", diff --git a/internal/message_equality.cc b/internal/message_equality.cc index 34080e6b1..921753222 100644 --- a/internal/message_equality.cc +++ b/internal/message_equality.cc @@ -462,7 +462,7 @@ absl::StatusOr MapValueAsEquatableValue( MessageFactory* absl_nonnull factory, EquatableValueReflection& reflection, const google::protobuf::MapValueConstRef& value, const FieldDescriptor* absl_nonnull field, std::string& scratch, - Unique& unpacked) { + google::protobuf::Arena::UniquePtr& unpacked) { if (IsAnyField(field)) { CEL_ASSIGN_OR_RETURN(unpacked, well_known_types::UnpackAnyIfResolveable( arena, reflection.any_reflection, @@ -516,7 +516,8 @@ absl::StatusOr RepeatedFieldAsEquatableValue( google::protobuf::Arena* absl_nonnull arena, const DescriptorPool* absl_nonnull pool, MessageFactory* absl_nonnull factory, EquatableValueReflection& reflection, const Message& message, const FieldDescriptor* absl_nonnull field, - int index, std::string& scratch, Unique& unpacked) { + int index, std::string& scratch, + google::protobuf::Arena::UniquePtr& unpacked) { if (IsAnyField(field)) { const auto& field_value = message.GetReflection()->GetRepeatedMessage(message, field, index); @@ -834,8 +835,8 @@ class MessageEqualsState final { auto rhs_well_known_type = rhs_descriptor->well_known_type(); const Message* absl_nonnull lhs_ptr = &lhs; const Message* absl_nonnull rhs_ptr = &rhs; - Unique lhs_unpacked; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; // Deal with any first. We could in theory check if we should bother // unpacking, but that is more complicated. We can always implement it // later. @@ -906,9 +907,9 @@ class MessageEqualsState final { } auto lhs_begin = ConstMapBegin(*lhs_reflection, lhs, *lhs_field); const auto lhs_end = ConstMapEnd(*lhs_reflection, lhs, *lhs_field); - Unique lhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; EquatableValue lhs_value; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; EquatableValue rhs_value; google::protobuf::MapKey rhs_map_key; google::protobuf::MapValueConstRef rhs_map_value; @@ -960,9 +961,9 @@ class MessageEqualsState final { if (size != rhs_reflection->FieldSize(rhs, rhs_field)) { return false; } - Unique lhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; EquatableValue lhs_value; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; EquatableValue rhs_value; for (int i = 0; i < size; ++i) { CEL_ASSIGN_OR_RETURN(lhs_value, @@ -1007,8 +1008,8 @@ class MessageEqualsState final { } const Message* absl_nonnull lhs_ptr = &lhs; const Message* absl_nonnull rhs_ptr = &rhs; - Unique lhs_unpacked; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; if (lhs_field != nullptr && IsAnyField(lhs_field)) { CEL_ASSIGN_OR_RETURN(lhs_unpacked, well_known_types::UnpackAnyIfResolveable( @@ -1093,7 +1094,7 @@ class MessageEqualsState final { return false; } const Message* absl_nullable rhs_packed = nullptr; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; if (rhs_field != nullptr && IsAnyField(rhs_field)) { rhs_packed = &rhs.GetReflection()->GetMessage(rhs, rhs_field); } else if (rhs_field == nullptr && IsAny(rhs)) { @@ -1174,7 +1175,7 @@ class MessageEqualsState final { return false; } const Message* absl_nullable lhs_packed = nullptr; - Unique lhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; if (lhs_field != nullptr && IsAnyField(lhs_field)) { lhs_packed = &lhs.GetReflection()->GetMessage(lhs, lhs_field); } else if (lhs_field == nullptr && IsAny(lhs)) { @@ -1262,7 +1263,7 @@ class MessageEqualsState final { return false; } const Message* absl_nullable rhs_packed = nullptr; - Unique rhs_unpacked; + google::protobuf::Arena::UniquePtr rhs_unpacked; if (rhs_field != nullptr && IsAnyField(rhs_field)) { rhs_packed = &rhs.GetReflection()->GetMessage(rhs, rhs_field); } else if (rhs_field == nullptr && IsAny(rhs)) { @@ -1342,7 +1343,7 @@ class MessageEqualsState final { return false; } const Message* absl_nullable lhs_packed = nullptr; - Unique lhs_unpacked; + google::protobuf::Arena::UniquePtr lhs_unpacked; if (lhs_field != nullptr && IsAnyField(lhs_field)) { lhs_packed = &lhs.GetReflection()->GetMessage(lhs, lhs_field); } else if (lhs_field == nullptr && IsAny(lhs)) { diff --git a/internal/message_equality_test.cc b/internal/message_equality_test.cc index 092edd71b..c08116181 100644 --- a/internal/message_equality_test.cc +++ b/internal/message_equality_test.cc @@ -14,6 +14,7 @@ #include "internal/message_equality.h" +#include #include #include #include @@ -27,12 +28,12 @@ #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/log/die_if_null.h" +#include "absl/memory/memory.h" #include "absl/status/status_matchers.h" #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/allocator.h" #include "common/memory.h" #include "internal/message_type_name.h" #include "internal/parse_text_proto.h" @@ -394,7 +395,7 @@ void PackMessageTo(const google::protobuf::Message& message, google::protobuf::M reflection.SetValue(instance, value); } -absl::optional, +absl::optional, const google::protobuf::FieldDescriptor* absl_nonnull>> PackTestAllTypesProto3Field(const google::protobuf::Message& message, const google::protobuf::FieldDescriptor* absl_nonnull field) { @@ -405,7 +406,7 @@ PackTestAllTypesProto3Field(const google::protobuf::Message& message, field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE) { const auto* descriptor = message.GetDescriptor(); const auto* any_field = descriptor->FindFieldByName("repeated_any"); - auto packed = WrapShared(message.New(), NewDeleteAllocator<>{}); + auto packed = absl::WrapUnique(message.New()); const int size = message.GetReflection()->FieldSize(message, field); for (int i = 0; i < size; ++i) { PackMessageTo( @@ -413,17 +414,17 @@ PackTestAllTypesProto3Field(const google::protobuf::Message& message, packed->GetReflection()->AddMessage(cel::to_address(packed), any_field)); } - return std::pair{packed, any_field}; + return std::pair{std::move(packed), any_field}; } if (!field->is_repeated() && field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE) { const auto* descriptor = message.GetDescriptor(); const auto* any_field = descriptor->FindFieldByName("single_any"); - auto packed = WrapShared(message.New(), NewDeleteAllocator<>{}); + auto packed = absl::WrapUnique(message.New()); PackMessageTo(message.GetReflection()->GetMessage(message, field), packed->GetReflection()->MutableMessage( cel::to_address(packed), any_field)); - return std::pair{packed, any_field}; + return std::pair{std::move(packed), any_field}; } return std::nullopt; } @@ -499,10 +500,10 @@ TEST_P(UnaryMessageFieldEqualsTest, Equals) { << rhs_field->name(); } // Test `google.protobuf.Any`. - absl::optional, + absl::optional, const google::protobuf::FieldDescriptor* absl_nonnull>> lhs_any = PackTestAllTypesProto3Field(*lhs_message, lhs_field); - absl::optional, + absl::optional, const google::protobuf::FieldDescriptor* absl_nonnull>> rhs_any = PackTestAllTypesProto3Field(*rhs_message, rhs_field); if (lhs_any) { diff --git a/internal/well_known_types.cc b/internal/well_known_types.cc index 5232bb6ad..f48d77f7f 100644 --- a/internal/well_known_types.cc +++ b/internal/well_known_types.cc @@ -1489,7 +1489,7 @@ google::protobuf::Message* absl_nonnull ValueReflection::MutableStructValue( return message->GetReflection()->MutableMessage(message, struct_value_field_); } -Unique ValueReflection::ReleaseListValue( +google::protobuf::Arena::UniquePtr ValueReflection::ReleaseListValue( google::protobuf::Message* absl_nonnull message) const { ABSL_DCHECK(IsInitialized()); ABSL_DCHECK_EQ(message->GetDescriptor(), descriptor_); @@ -1497,12 +1497,12 @@ Unique ValueReflection::ReleaseListValue( if (!reflection->HasField(*message, list_value_field_)) { reflection->MutableMessage(message, list_value_field_); } - return WrapUnique( - reflection->UnsafeArenaReleaseMessage(message, list_value_field_), - message->GetArena()); + return google::protobuf::Arena::UnsafeWrapUniquePtr( + message->GetArena(), + reflection->UnsafeArenaReleaseMessage(message, list_value_field_)); } -Unique ValueReflection::ReleaseStructValue( +google::protobuf::Arena::UniquePtr ValueReflection::ReleaseStructValue( google::protobuf::Message* absl_nonnull message) const { ABSL_DCHECK(IsInitialized()); ABSL_DCHECK_EQ(message->GetDescriptor(), descriptor_); @@ -1510,9 +1510,9 @@ Unique ValueReflection::ReleaseStructValue( if (!reflection->HasField(*message, struct_value_field_)) { reflection->MutableMessage(message, struct_value_field_); } - return WrapUnique( - reflection->UnsafeArenaReleaseMessage(message, struct_value_field_), - message->GetArena()); + return google::protobuf::Arena::UnsafeWrapUniquePtr( + message->GetArena(), + reflection->UnsafeArenaReleaseMessage(message, struct_value_field_)); } absl::StatusOr GetValueReflection( @@ -1899,9 +1899,9 @@ namespace { // it as `ListValue`. If adapted is empty, we return as a reference. If adapted // is present, message must be a reference to the value held in adapted and it // will be returned by value. -absl::StatusOr AdaptListValue(google::protobuf::Arena* absl_nullable arena, - const google::protobuf::Message& message, - Unique adapted) { +absl::StatusOr AdaptListValue( + google::protobuf::Arena* absl_nullable arena, const google::protobuf::Message& message, + google::protobuf::Arena::UniquePtr adapted) { ABSL_DCHECK(!adapted || &message == cel::to_address(adapted)); const auto* descriptor = message.GetDescriptor(); if (ABSL_PREDICT_FALSE(descriptor == nullptr)) { @@ -1922,9 +1922,9 @@ absl::StatusOr AdaptListValue(google::protobuf::Arena* absl_nullable // as `Struct`. If adapted is empty, we return as a reference. If adapted is // present, message must be a reference to the value held in adapted and it will // be returned by value. -absl::StatusOr AdaptStruct(google::protobuf::Arena* absl_nullable arena, - const google::protobuf::Message& message, - Unique adapted) { +absl::StatusOr AdaptStruct( + google::protobuf::Arena* absl_nullable arena, const google::protobuf::Message& message, + google::protobuf::Arena::UniquePtr adapted) { ABSL_DCHECK(!adapted || &message == cel::to_address(adapted)); const auto* descriptor = message.GetDescriptor(); if (ABSL_PREDICT_FALSE(descriptor == nullptr)) { @@ -1942,14 +1942,14 @@ absl::StatusOr AdaptStruct(google::protobuf::Arena* absl_nullable arena, // AdaptAny recursively unpacks a protocol buffer message which is an instance // of `google.protobuf.Any`. -absl::StatusOr> AdaptAny( +absl::StatusOr> AdaptAny( google::protobuf::Arena* absl_nullable arena, AnyReflection& reflection, const google::protobuf::Message& message, const Descriptor* absl_nonnull descriptor, const DescriptorPool* absl_nonnull pool, google::protobuf::MessageFactory* absl_nonnull factory, bool error_if_unresolveable) { ABSL_DCHECK_EQ(descriptor->well_known_type(), Descriptor::WELLKNOWNTYPE_ANY); const google::protobuf::Message* absl_nonnull to_unwrap = &message; - Unique unwrapped; + google::protobuf::Arena::UniquePtr unwrapped; std::string type_url_scratch; std::string value_scratch; do { @@ -1979,7 +1979,8 @@ absl::StatusOr> AdaptAny( "unable to build prototype for type name: ", type_url_view)); } BytesValue value = reflection.GetValue(*to_unwrap, value_scratch); - Unique unpacked = WrapUnique(prototype->New(arena), arena); + google::protobuf::Arena::UniquePtr unpacked = + google::protobuf::Arena::UnsafeWrapUniquePtr(arena, prototype->New(arena)); // TODO(b/557267722): Extensions that are not included in the same // descriptor pool as the resolved descriptor will be treated as unknown // fields. Extending messages like this should be exceedingly rare and is @@ -2014,7 +2015,7 @@ absl::StatusOr> AdaptAny( } // namespace -absl::StatusOr> UnpackAnyFrom( +absl::StatusOr> UnpackAnyFrom( google::protobuf::Arena* absl_nullable arena, AnyReflection& reflection, const google::protobuf::Message& message, const google::protobuf::DescriptorPool* absl_nonnull pool, @@ -2025,11 +2026,12 @@ absl::StatusOr> UnpackAnyFrom( factory, /*error_if_unresolveable=*/true); } -absl::StatusOr> UnpackAnyIfResolveable( - google::protobuf::Arena* absl_nullable arena, AnyReflection& reflection, - const google::protobuf::Message& message, - const google::protobuf::DescriptorPool* absl_nonnull pool, - google::protobuf::MessageFactory* absl_nonnull factory) { +absl::StatusOr> +UnpackAnyIfResolveable(google::protobuf::Arena* absl_nullable arena, + AnyReflection& reflection, + const google::protobuf::Message& message, + const google::protobuf::DescriptorPool* absl_nonnull pool, + google::protobuf::MessageFactory* absl_nonnull factory) { ABSL_DCHECK_EQ(message.GetDescriptor()->well_known_type(), Descriptor::WELLKNOWNTYPE_ANY); return AdaptAny(arena, reflection, message, message.GetDescriptor(), pool, @@ -2047,7 +2049,7 @@ absl::StatusOr AdaptFromMessage( message.GetTypeName())); } const google::protobuf::Message* absl_nonnull to_adapt; - Unique adapted; + google::protobuf::Arena::UniquePtr adapted; Descriptor::WellKnownType well_known_type = descriptor->well_known_type(); if (well_known_type == Descriptor::WELLKNOWNTYPE_ANY) { AnyReflection reflection; diff --git a/internal/well_known_types.h b/internal/well_known_types.h index f63e5e76b..3e09b286b 100644 --- a/internal/well_known_types.h +++ b/internal/well_known_types.h @@ -50,7 +50,6 @@ #include "absl/time/time.h" #include "absl/types/variant.h" #include "common/any.h" -#include "common/memory.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/map_field.h" @@ -1007,10 +1006,10 @@ class ValueReflection final { google::protobuf::Message* absl_nonnull MutableStructValue( google::protobuf::Message* absl_nonnull message) const; - Unique ReleaseListValue( + google::protobuf::Arena::UniquePtr ReleaseListValue( google::protobuf::Message* absl_nonnull message) const; - Unique ReleaseStructValue( + google::protobuf::Arena::UniquePtr ReleaseStructValue( google::protobuf::Message* absl_nonnull message) const; private: @@ -1278,18 +1277,18 @@ absl::StatusOr GetFieldMaskReflection( const google::protobuf::Descriptor* absl_nonnull descriptor ABSL_ATTRIBUTE_LIFETIME_BOUND); -using ListValuePtr = Unique; +using ListValuePtr = google::protobuf::Arena::UniquePtr; using ListValueConstRef = std::reference_wrapper; -using StructPtr = Unique; +using StructPtr = google::protobuf::Arena::UniquePtr; using StructConstRef = std::reference_wrapper; // Variant holding `std::reference_wrapper` or `Unique`, either of which is an -// instance of `google.protobuf.ListValue` which is either a generated message -// or dynamic message. +// google::protobuf::Message>` or `google::protobuf::Arena::UniquePtr`, either of +// which is an instance of `google.protobuf.ListValue` which is either a +// generated message or dynamic message. class ListValue final : public absl::variant { using absl::variant::variant; }; @@ -1316,9 +1315,9 @@ inline absl::variant&& AsVariant( } // Variant holding `std::reference_wrapper` or `Unique`, either of which is an -// instance of `google.protobuf.Struct` which is either a generated message or -// dynamic message. +// google::protobuf::Message>` or `google::protobuf::Arena::UniquePtr`, either of +// which is an instance of `google.protobuf.Struct` which is either a generated +// message or dynamic message. class Struct final : public absl::variant { public: using absl::variant::variant; @@ -1342,13 +1341,14 @@ inline absl::variant&& AsVariant(Struct&& value) { } // Variant capable of representing any unwrapped well known type or message. -using Value = absl::variant>; +using Value = + absl::variant>; // Unpacks the given instance of `google.protobuf.Any`. -absl::StatusOr> UnpackAnyFrom( +absl::StatusOr> UnpackAnyFrom( google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND, AnyReflection& reflection, const google::protobuf::Message& message, const google::protobuf::DescriptorPool* absl_nonnull pool @@ -1356,7 +1356,8 @@ absl::StatusOr> UnpackAnyFrom( google::protobuf::MessageFactory* absl_nonnull factory ABSL_ATTRIBUTE_LIFETIME_BOUND); // Unpacks the given instance of `google.protobuf.Any` if it is resolvable. -absl::StatusOr> UnpackAnyIfResolveable( +absl::StatusOr> +UnpackAnyIfResolveable( google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND, AnyReflection& reflection, const google::protobuf::Message& message, const google::protobuf::DescriptorPool* absl_nonnull pool diff --git a/internal/well_known_types_test.cc b/internal/well_known_types_test.cc index 5ffc06ee9..aae66c51c 100644 --- a/internal/well_known_types_test.cc +++ b/internal/well_known_types_test.cc @@ -34,8 +34,6 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/variant.h" -#include "common/memory.h" #include "internal/message_type_name.h" #include "internal/minimal_descriptor_pool.h" #include "internal/parse_text_proto.h" @@ -1027,8 +1025,10 @@ TEST_F(AdaptFromMessageTest, Any_Struct) { TEST_F(AdaptFromMessageTest, Any_TestAllTypesProto3) { auto message = DynamicParseTextProto( R"pb(type_url: "type.googleapis.com/cel.expr.conformance.proto3.TestAllTypes")pb"); - EXPECT_THAT(AdaptFromMessage(*message), - IsOkAndHolds(VariantWith>(NotNull()))); + EXPECT_THAT( + AdaptFromMessage(*message), + IsOkAndHolds( + VariantWith>(NotNull()))); } TEST_F(AdaptFromMessageTest, Any_BadTypeUrlDomain) { From 9e75d37ab393075c98d0c1b9e596d633530a305c Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 6 Oct 2026 05:22:31 -0700 Subject: [PATCH 263/350] Begin getting rid of `cel::MemoryManager` PiperOrigin-RevId: 994328120 --- common/BUILD | 1 - common/memory.cc | 9 --- common/memory.h | 11 ---- eval/public/BUILD | 1 - eval/public/cel_value.cc | 52 ---------------- eval/public/cel_value.h | 72 +++++++++++++--------- extensions/protobuf/memory_manager.cc | 3 +- extensions/protobuf/memory_manager_test.cc | 6 -- 8 files changed, 44 insertions(+), 111 deletions(-) diff --git a/common/BUILD b/common/BUILD index f75c82f17..a41f0cb1a 100644 --- a/common/BUILD +++ b/common/BUILD @@ -506,7 +506,6 @@ cc_library( ":allocator", "//internal:to_address", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/numeric:bits", diff --git a/common/memory.cc b/common/memory.cc index c00c12ed8..46373ba5b 100644 --- a/common/memory.cc +++ b/common/memory.cc @@ -19,10 +19,8 @@ #include #include -#include "absl/base/no_destructor.h" #include "absl/log/absl_check.h" #include "absl/numeric/bits.h" -#include "google/protobuf/arena.h" namespace cel { @@ -73,11 +71,4 @@ bool ReferenceCountingMemoryManager::Deallocate(void* ptr, size_t size, return true; } -MemoryManager MemoryManager::Unmanaged() { - // A static singleton arena, using `absl::NoDestructor` to avoid warnings - // related static variables without trivial destructors. - static absl::NoDestructor arena; - return MemoryManager::Pooling(&*arena); -} - } // namespace cel diff --git a/common/memory.h b/common/memory.h index 967bc57e2..a9ae46023 100644 --- a/common/memory.h +++ b/common/memory.h @@ -152,17 +152,6 @@ class PoolingMemoryManager final { // resources. class MemoryManager final { public: - // Returns a `MemoryManager` which utilizes an arena but never frees its - // memory. It is effectively a memory leak and should only be used for limited - // use cases, such as initializing singletons which live for the life of the - // program. - static MemoryManager Unmanaged(); - - // Returns a `MemoryManager` which utilizes reference counting. - ABSL_MUST_USE_RESULT static MemoryManager ReferenceCounting() { - return MemoryManager(nullptr); - } - // Returns a `MemoryManager` which utilizes an arena. ABSL_MUST_USE_RESULT static MemoryManager Pooling( google::protobuf::Arena* absl_nonnull arena) { diff --git a/eval/public/BUILD b/eval/public/BUILD index 2172774a9..f432767fa 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -96,7 +96,6 @@ cc_library( "//common:native_type", "//eval/internal:errors", "//eval/public/structs:legacy_type_info_apis", - "//extensions/protobuf:memory_manager", "//internal:casts", "//internal:status_macros", "//internal:utf8", diff --git a/eval/public/cel_value.cc b/eval/public/cel_value.cc index 7e2d3b5d4..33b78c1c9 100644 --- a/eval/public/cel_value.cc +++ b/eval/public/cel_value.cc @@ -14,10 +14,8 @@ #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/memory.h" #include "eval/internal/errors.h" #include "eval/public/structs/legacy_type_info_apis.h" -#include "extensions/protobuf/memory_manager.h" #include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -286,23 +284,6 @@ CelValue CelValue::CreateList() { return CreateList(EmptyCelList::Get()); } CelValue CelValue::CreateMap() { return CreateMap(EmptyCelMap::Get()); } -CelValue CreateErrorValue(cel::MemoryManagerRef manager, - absl::string_view message, - absl::StatusCode error_code) { - // TODO(uncreated-issue/1): assume arena-style allocator while migrating to new - // value type. - Arena* arena = cel::extensions::ProtoMemoryManagerArena(manager); - return CreateErrorValue(arena, message, error_code); -} - -CelValue CreateErrorValue(cel::MemoryManagerRef manager, - const absl::Status& status) { - // TODO(uncreated-issue/1): assume arena-style allocator while migrating to new - // value type. - Arena* arena = cel::extensions::ProtoMemoryManagerArena(manager); - return CreateErrorValue(arena, status); -} - CelValue CreateErrorValue(Arena* arena, absl::string_view message, absl::StatusCode error_code) { CelError* error = Arena::Create(arena, error_code, message); @@ -314,12 +295,6 @@ CelValue CreateErrorValue(Arena* arena, const absl::Status& status) { return CelValue::CreateError(error); } -CelValue CreateNoMatchingOverloadError(cel::MemoryManagerRef manager, - absl::string_view fn) { - return CelValue::CreateError(interop::CreateNoMatchingOverloadError( - cel::extensions::ProtoMemoryManagerArena(manager), fn)); -} - CelValue CreateNoMatchingOverloadError(google::protobuf::Arena* arena, absl::string_view fn) { return CelValue::CreateError( @@ -333,22 +308,10 @@ bool CheckNoMatchingOverloadError(CelValue value) { cel::runtime_internal::kErrNoMatchingOverload); } -CelValue CreateNoSuchFieldError(cel::MemoryManagerRef manager, - absl::string_view field) { - return CelValue::CreateError(interop::CreateNoSuchFieldError( - cel::extensions::ProtoMemoryManagerArena(manager), field)); -} - CelValue CreateNoSuchFieldError(google::protobuf::Arena* arena, absl::string_view field) { return CelValue::CreateError(interop::CreateNoSuchFieldError(arena, field)); } -CelValue CreateNoSuchKeyError(cel::MemoryManagerRef manager, - absl::string_view key) { - return CelValue::CreateError(interop::CreateNoSuchKeyError( - cel::extensions::ProtoMemoryManagerArena(manager), key)); -} - CelValue CreateNoSuchKeyError(google::protobuf::Arena* arena, absl::string_view key) { return CelValue::CreateError(interop::CreateNoSuchKeyError(arena, key)); } @@ -365,15 +328,6 @@ CelValue CreateMissingAttributeError(google::protobuf::Arena* arena, interop::CreateMissingAttributeError(arena, missing_attribute_path)); } -CelValue CreateMissingAttributeError(cel::MemoryManagerRef manager, - absl::string_view missing_attribute_path) { - // TODO(uncreated-issue/1): assume arena-style allocator while migrating - // to new value type. - return CelValue::CreateError(interop::CreateMissingAttributeError( - cel::extensions::ProtoMemoryManagerArena(manager), - missing_attribute_path)); -} - bool IsMissingAttributeError(const CelValue& value) { const CelError* error; if (!value.GetValue(&error)) return false; @@ -385,12 +339,6 @@ bool IsMissingAttributeError(const CelValue& value) { return false; } -CelValue CreateUnknownFunctionResultError(cel::MemoryManagerRef manager, - absl::string_view help_message) { - return CelValue::CreateError(interop::CreateUnknownFunctionResultError( - cel::extensions::ProtoMemoryManagerArena(manager), help_message)); -} - CelValue CreateUnknownFunctionResultError(google::protobuf::Arena* arena, absl::string_view help_message) { return CelValue::CreateError( diff --git a/eval/public/cel_value.h b/eval/public/cel_value.h index 5bf50d01a..2a98414f3 100644 --- a/eval/public/cel_value.h +++ b/eval/public/cel_value.h @@ -653,44 +653,54 @@ class CelMap { // Utility method that generates CelValue containing CelError. // message an error message // error_code error code -CelValue CreateErrorValue( - cel::MemoryManagerRef manager ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view message, - absl::StatusCode error_code = absl::StatusCode::kUnknown); CelValue CreateErrorValue( google::protobuf::Arena* arena, absl::string_view message, absl::StatusCode error_code = absl::StatusCode::kUnknown); - -// Utility method for generating a CelValue from an absl::Status. -CelValue CreateErrorValue(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - const absl::Status& status); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateErrorValue( + cel::MemoryManagerRef manager ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view message, + absl::StatusCode error_code = absl::StatusCode::kUnknown) { + return CreateErrorValue(manager.arena(), message, error_code); +} // Utility method for generating a CelValue from an absl::Status. CelValue CreateErrorValue(google::protobuf::Arena* arena, const absl::Status& status); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateErrorValue(cel::MemoryManagerRef manager + ABSL_ATTRIBUTE_LIFETIME_BOUND, + const absl::Status& status) { + return CreateErrorValue(manager.arena(), status); +} // Create an error for failed overload resolution, optionally including the name // of the function. -CelValue CreateNoMatchingOverloadError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view fn = ""); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateNoMatchingOverloadError(google::protobuf::Arena* arena, absl::string_view fn = ""); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateNoMatchingOverloadError(cel::MemoryManagerRef manager + ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view fn = "") { + return CreateNoMatchingOverloadError(manager.arena(), fn); +} bool CheckNoMatchingOverloadError(CelValue value); -CelValue CreateNoSuchFieldError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view field = ""); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateNoSuchFieldError(google::protobuf::Arena* arena, absl::string_view field = ""); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateNoSuchFieldError(cel::MemoryManagerRef manager + ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view field = "") { + return CreateNoSuchFieldError(manager.arena(), field); +} -CelValue CreateNoSuchKeyError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view key); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateNoSuchKeyError(google::protobuf::Arena* arena, absl::string_view key); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateNoSuchKeyError(cel::MemoryManagerRef manager + ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view key) { + return CreateNoSuchKeyError(manager.arena(), key); +} bool CheckNoSuchKeyError(CelValue value); @@ -698,12 +708,14 @@ bool CheckNoSuchKeyError(CelValue value); // value is undefined. For example, this may represent a field in a proto // message bound to the activation whose value can't be determined by the // hosting application. -CelValue CreateMissingAttributeError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view missing_attribute_path); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateMissingAttributeError(google::protobuf::Arena* arena, absl::string_view missing_attribute_path); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateMissingAttributeError( + cel::MemoryManagerRef manager ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view missing_attribute_path) { + return CreateMissingAttributeError(manager.arena(), missing_attribute_path); +} ABSL_CONST_INIT extern const absl::string_view kPayloadUrlMissingAttributePath; bool IsMissingAttributeError(const CelValue& value); @@ -711,12 +723,14 @@ bool IsMissingAttributeError(const CelValue& value); // Returns error indicating the result of the function is unknown. This is used // as a signal to create an unknown set if unknown function handling is opted // into. -CelValue CreateUnknownFunctionResultError(cel::MemoryManagerRef manager - ABSL_ATTRIBUTE_LIFETIME_BOUND, - absl::string_view help_message); -ABSL_DEPRECATED("Prefer using the generic MemoryManager overload") CelValue CreateUnknownFunctionResultError(google::protobuf::Arena* arena, absl::string_view help_message); +ABSL_DEPRECATE_AND_INLINE() +inline CelValue CreateUnknownFunctionResultError( + cel::MemoryManagerRef manager ABSL_ATTRIBUTE_LIFETIME_BOUND, + absl::string_view help_message) { + return CreateUnknownFunctionResultError(manager.arena(), help_message); +} // Returns true if this is unknown value error indicating that evaluation // called an extension function whose value is unknown for the given args. diff --git a/extensions/protobuf/memory_manager.cc b/extensions/protobuf/memory_manager.cc index 5b3e6e74b..0953d0a45 100644 --- a/extensions/protobuf/memory_manager.cc +++ b/extensions/protobuf/memory_manager.cc @@ -23,8 +23,7 @@ namespace cel { namespace extensions { MemoryManagerRef ProtoMemoryManager(google::protobuf::Arena* arena) { - return arena != nullptr ? MemoryManagerRef::Pooling(arena) - : MemoryManagerRef::ReferenceCounting(); + return MemoryManagerRef::Pooling(arena); } google::protobuf::Arena* absl_nullable ProtoMemoryManagerArena( diff --git a/extensions/protobuf/memory_manager_test.cc b/extensions/protobuf/memory_manager_test.cc index ddab4cf32..956842bb9 100644 --- a/extensions/protobuf/memory_manager_test.cc +++ b/extensions/protobuf/memory_manager_test.cc @@ -22,7 +22,6 @@ namespace cel::extensions { namespace { using ::testing::Eq; -using ::testing::IsNull; using ::testing::NotNull; TEST(ProtoMemoryManager, MemoryManagement) { @@ -41,17 +40,12 @@ TEST(ProtoMemoryManagerRef, MemoryManagement) { google::protobuf::Arena arena; auto memory_manager = ProtoMemoryManagerRef(&arena); EXPECT_EQ(memory_manager.memory_management(), MemoryManagement::kPooling); - memory_manager = ProtoMemoryManagerRef(nullptr); - EXPECT_EQ(memory_manager.memory_management(), - MemoryManagement::kReferenceCounting); } TEST(ProtoMemoryManagerRef, Arena) { google::protobuf::Arena arena; auto memory_manager = ProtoMemoryManagerRef(&arena); EXPECT_THAT(ProtoMemoryManagerArena(memory_manager), Eq(&arena)); - memory_manager = ProtoMemoryManagerRef(nullptr); - EXPECT_THAT(ProtoMemoryManagerArena(memory_manager), IsNull()); } } // namespace From df93dfaa54b2c578cf69891675a14097032e1ea6 Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 6 Oct 2026 11:04:41 -0700 Subject: [PATCH 264/350] Workaround Clang-CL compiler bug PiperOrigin-RevId: 994503537 --- internal/BUILD | 1 - internal/message_equality.cc | 29 ++++++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/internal/BUILD b/internal/BUILD index e49dd67e3..b6cabfb75 100644 --- a/internal/BUILD +++ b/internal/BUILD @@ -811,7 +811,6 @@ cc_library( ":number", ":status_macros", ":well_known_types", - "//common:memory", "//extensions/protobuf/internal:map_reflection", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", diff --git a/internal/message_equality.cc b/internal/message_equality.cc index 921753222..bfa12883c 100644 --- a/internal/message_equality.cc +++ b/internal/message_equality.cc @@ -33,7 +33,6 @@ #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "absl/types/variant.h" -#include "common/memory.h" #include "extensions/protobuf/internal/map_reflection.h" #include "internal/json.h" #include "internal/number.h" @@ -846,7 +845,7 @@ class MessageEqualsState final { well_known_types::UnpackAnyIfResolveable( &arena_, lhs_reflection_.any_reflection, lhs, pool_, factory_)); if (lhs_unpacked) { - lhs_ptr = cel::to_address(lhs_unpacked); + lhs_ptr = lhs_unpacked.get(); lhs_descriptor = lhs_ptr->GetDescriptor(); lhs_well_known_type = lhs_descriptor->well_known_type(); } @@ -857,7 +856,7 @@ class MessageEqualsState final { well_known_types::UnpackAnyIfResolveable( &arena_, rhs_reflection_.any_reflection, rhs, pool_, factory_)); if (rhs_unpacked) { - rhs_ptr = cel::to_address(rhs_unpacked); + rhs_ptr = rhs_unpacked.get(); rhs_descriptor = rhs_ptr->GetDescriptor(); rhs_well_known_type = rhs_descriptor->well_known_type(); } @@ -1017,7 +1016,7 @@ class MessageEqualsState final { lhs.GetReflection()->GetMessage(lhs, lhs_field), pool_, factory_)); if (lhs_unpacked) { - lhs_ptr = cel::to_address(lhs_unpacked); + lhs_ptr = lhs_unpacked.get(); lhs_field = nullptr; } } else if (lhs_field == nullptr && IsAny(lhs)) { @@ -1026,7 +1025,7 @@ class MessageEqualsState final { well_known_types::UnpackAnyIfResolveable( &arena_, lhs_reflection_.any_reflection, lhs, pool_, factory_)); if (lhs_unpacked) { - lhs_ptr = cel::to_address(lhs_unpacked); + lhs_ptr = lhs_unpacked.get(); } } if (rhs_field != nullptr && IsAnyField(rhs_field)) { @@ -1036,7 +1035,7 @@ class MessageEqualsState final { rhs.GetReflection()->GetMessage(rhs, rhs_field), pool_, factory_)); if (rhs_unpacked) { - rhs_ptr = cel::to_address(rhs_unpacked); + rhs_ptr = rhs_unpacked.get(); rhs_field = nullptr; } } else if (rhs_field == nullptr && IsAny(rhs)) { @@ -1045,7 +1044,7 @@ class MessageEqualsState final { well_known_types::UnpackAnyIfResolveable( &arena_, rhs_reflection_.any_reflection, rhs, pool_, factory_)); if (rhs_unpacked) { - rhs_ptr = cel::to_address(rhs_unpacked); + rhs_ptr = rhs_unpacked.get(); } } EquatableValue lhs_value; @@ -1125,8 +1124,8 @@ class MessageEqualsState final { const Message* absl_nonnull rhs_message = rhs_field != nullptr ? &rhs.GetReflection()->GetMessage(rhs, rhs_field) - : rhs_unpacked != nullptr ? cel::to_address(rhs_unpacked) - : &rhs; + : rhs_unpacked.get() != nullptr ? rhs_unpacked.get() + : &rhs; const auto* rhs_descriptor = rhs_message->GetDescriptor(); const auto rhs_well_known_type = rhs_descriptor->well_known_type(); switch (rhs_well_known_type) { @@ -1206,8 +1205,8 @@ class MessageEqualsState final { const Message* absl_nonnull lhs_message = lhs_field != nullptr ? &lhs.GetReflection()->GetMessage(lhs, lhs_field) - : lhs_unpacked != nullptr ? cel::to_address(lhs_unpacked) - : &lhs; + : lhs_unpacked.get() != nullptr ? lhs_unpacked.get() + : &lhs; const auto* lhs_descriptor = lhs_message->GetDescriptor(); const auto lhs_well_known_type = lhs_descriptor->well_known_type(); switch (lhs_well_known_type) { @@ -1294,8 +1293,8 @@ class MessageEqualsState final { const Message* absl_nonnull rhs_message = rhs_field != nullptr ? &rhs.GetReflection()->GetMessage(rhs, rhs_field) - : rhs_unpacked != nullptr ? cel::to_address(rhs_unpacked) - : &rhs; + : rhs_unpacked.get() != nullptr ? rhs_unpacked.get() + : &rhs; const auto* rhs_descriptor = rhs_message->GetDescriptor(); const auto rhs_well_known_type = rhs_descriptor->well_known_type(); switch (rhs_well_known_type) { @@ -1374,8 +1373,8 @@ class MessageEqualsState final { const Message* absl_nonnull lhs_message = lhs_field != nullptr ? &lhs.GetReflection()->GetMessage(lhs, lhs_field) - : lhs_unpacked != nullptr ? cel::to_address(lhs_unpacked) - : &lhs; + : lhs_unpacked.get() != nullptr ? lhs_unpacked.get() + : &lhs; const auto* lhs_descriptor = lhs_message->GetDescriptor(); const auto lhs_well_known_type = lhs_descriptor->well_known_type(); switch (lhs_well_known_type) { From 0f312ae9f7f9ceb67a83a9efc0b1624745003399 Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 6 Oct 2026 11:21:45 -0700 Subject: [PATCH 265/350] Update string and bytes literal formatting to support appending without flattening PiperOrigin-RevId: 994514844 --- internal/strings.cc | 310 +++++++++++++++++++++++++++------------ internal/strings.h | 11 ++ internal/strings_test.cc | 26 +++- 3 files changed, 254 insertions(+), 93 deletions(-) diff --git a/internal/strings.cc b/internal/strings.cc index a272aaa46..144ae91de 100644 --- a/internal/strings.cc +++ b/internal/strings.cc @@ -14,13 +14,13 @@ #include "internal/strings.h" +#include #include #include "absl/base/attributes.h" #include "absl/status/status.h" #include "absl/strings/ascii.h" #include "absl/strings/cord.h" -#include "absl/strings/escaping.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" @@ -441,62 +441,100 @@ bool UnescapeInternal(absl::string_view source, absl::string_view closing_str, return true; } -std::string EscapeInternal(absl::string_view src, bool escape_all_bytes, - char escape_quote_char) { - std::string dest; - // Worst case size is every byte has to be hex escaped, so 4 char for every - // byte. - dest.reserve(src.size() * 4); - bool last_hex_escape = false; // true if last output char was \xNN. - const char* p = src.data(); - const char* end = p + src.size(); - for (; p < end; ++p) { - unsigned char c = static_cast(*p); - bool is_hex_escape = false; - switch (c) { - case '\n': - dest.append("\\n"); - break; - case '\r': - dest.append("\\r"); - break; - case '\t': - dest.append("\\t"); +[[nodiscard]] +bool EscapeChar(std::string* dest, unsigned char c, bool escape_all_bytes, + char escape_quote_char, bool last_hex_escape) { + bool is_hex_escape = false; + switch (c) { + case '\n': + dest->append("\\n"); + break; + case '\r': + dest->append("\\r"); + break; + case '\t': + dest->append("\\t"); + break; + case '\\': + dest->append("\\\\"); + break; + case '\'': + ABSL_FALLTHROUGH_INTENDED; + case '\"': + ABSL_FALLTHROUGH_INTENDED; + case '`': + // Escape only quote chars that match escape_quote_char. + if (escape_quote_char == 0 || c == escape_quote_char) { + dest->push_back('\\'); + } + dest->push_back(c); + break; + default: + // Note that if we emit \xNN and the src character after that is a hex + // digit then that digit must be escaped too to prevent it being + // interpreted as part of the character code by C. + if ((!escape_all_bytes || c < 0x80) && + (!absl::ascii_isprint(c) || + (last_hex_escape && absl::ascii_isxdigit(c)))) { + dest->append("\\x"); + dest->push_back(kHexTable[c / 16]); + dest->push_back(kHexTable[c % 16]); + is_hex_escape = true; + } else { + dest->push_back(c); break; + } + } + return is_hex_escape; +} + +template +void EscapeStringInternal(std::string* dest, Iter src_begin, const Iter src_end, + bool escape_all_bytes, char escape_quote_char) { + bool last_hex_escape = false; // true if last output char was \xNN. + for (; src_begin != src_end; ++src_begin) { + last_hex_escape = + EscapeChar(dest, static_cast(*src_begin), + escape_all_bytes, escape_quote_char, last_hex_escape); + } +} + +void EscapeByte(std::string* dest, unsigned char b, bool escape_all_bytes, + char escape_quote_char) { + if (escape_all_bytes || !absl::ascii_isprint(b)) { + dest->append("\\x"); + dest->push_back(kHexTable[b / 16]); + dest->push_back(kHexTable[b % 16]); + } else { + switch (b) { + // Note that we only handle printable escape characters here. All + // unprintable (\n, \r, \t, etc.) are hex escaped above. case '\\': - dest.append("\\\\"); + dest->append("\\\\"); break; case '\'': - ABSL_FALLTHROUGH_INTENDED; - case '\"': - ABSL_FALLTHROUGH_INTENDED; + case '"': case '`': // Escape only quote chars that match escape_quote_char. - if (escape_quote_char == 0 || c == escape_quote_char) { - dest.push_back('\\'); + if (escape_quote_char == 0 || b == escape_quote_char) { + dest->push_back('\\'); } - dest.push_back(c); + dest->push_back(static_cast(b)); break; default: - // Note that if we emit \xNN and the src character after that is a hex - // digit then that digit must be escaped too to prevent it being - // interpreted as part of the character code by C. - if ((!escape_all_bytes || c < 0x80) && - (!absl::ascii_isprint(c) || - (last_hex_escape && absl::ascii_isxdigit(c)))) { - dest.append("\\x"); - dest.push_back(kHexTable[c / 16]); - dest.push_back(kHexTable[c % 16]); - is_hex_escape = true; - } else { - dest.push_back(c); - break; - } + dest->push_back(static_cast(b)); + break; } - last_hex_escape = is_hex_escape; } - dest.shrink_to_fit(); - return dest; +} + +template +void EscapeBytesInternal(std::string* dest, Iter src_begin, const Iter src_end, + bool escape_all_bytes, char escape_quote_char) { + for (; src_begin != src_end; ++src_begin) { + EscapeByte(dest, static_cast(*src_begin), escape_all_bytes, + escape_quote_char); + } } bool MayBeTripleQuotedString(absl::string_view str) { @@ -550,42 +588,25 @@ absl::StatusOr UnescapeBytes(absl::string_view str) { } std::string EscapeString(absl::string_view str) { - return EscapeInternal(str, true, '\0'); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4); + EscapeStringInternal(&dest, str.begin(), str.end(), true, '\0'); + dest.shrink_to_fit(); + return dest; } std::string EscapeBytes(absl::string_view str, bool escape_all_bytes, char escape_quote_char) { - std::string escaped_bytes; - const char* p = str.data(); - const char* end = p + str.size(); - for (; p < end; ++p) { - unsigned char c = *p; - if (escape_all_bytes || !absl::ascii_isprint(c)) { - escaped_bytes += "\\x"; - escaped_bytes += absl::BytesToHexString(absl::string_view(p, 1)); - } else { - switch (c) { - // Note that we only handle printable escape characters here. All - // unprintable (\n, \r, \t, etc.) are hex escaped above. - case '\\': - escaped_bytes += "\\\\"; - break; - case '\'': - case '"': - case '`': - // Escape only quote chars that match escape_quote_char. - if (escape_quote_char == 0 || c == escape_quote_char) { - escaped_bytes += '\\'; - } - escaped_bytes += c; - break; - default: - escaped_bytes += c; - break; - } - } - } - return escaped_bytes; + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4); + EscapeBytesInternal(&dest, str.begin(), str.end(), escape_all_bytes, + escape_quote_char); + dest.shrink_to_fit(); + return dest; } absl::StatusOr ParseStringLiteral(absl::string_view str) { @@ -649,38 +670,143 @@ absl::StatusOr ParseBytesLiteral(absl::string_view str) { } std::string FormatStringLiteral(absl::string_view str) { - absl::string_view quote = - (str.find('"') != str.npos && str.find('\'') == str.npos) ? "'" : "\""; - return absl::StrCat(quote, EscapeInternal(str, true, quote[0]), quote); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 2); + FormatStringLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; } std::string FormatStringLiteral(const absl::Cord& str) { - if (auto flat = str.TryFlat(); flat) { - return FormatStringLiteral(*flat); - } - return FormatStringLiteral(static_cast(str)); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 2); + FormatStringLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatStringLiteralTo(absl::string_view str, std::string* dest) { + const char quote = + absl::StrContains(str, '"') && !absl::StrContains(str, '\'') ? '\'' : '"'; + dest->push_back(quote); + EscapeStringInternal(dest, str.begin(), str.end(), true, quote); + dest->push_back(quote); +} + +void FormatStringLiteralTo(const absl::Cord& str, std::string* dest) { + const char quote = (str.Contains("\"") && !str.Contains("'")) ? '\'' : '"'; + dest->push_back(quote); + EscapeStringInternal(dest, str.char_begin(), str.char_end(), true, quote); + dest->push_back(quote); } std::string FormatSingleQuotedStringLiteral(absl::string_view str) { - return absl::StrCat("'", EscapeInternal(str, true, '\''), "'"); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 2); + FormatSingleQuotedStringLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatSingleQuotedStringLiteralTo(absl::string_view str, + std::string* dest) { + dest->push_back('\''); + EscapeStringInternal(dest, str.begin(), str.end(), true, '\''); + dest->push_back('\''); } std::string FormatDoubleQuotedStringLiteral(absl::string_view str) { - return absl::StrCat("\"", EscapeInternal(str, true, '"'), "\""); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 2); + FormatDoubleQuotedStringLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatDoubleQuotedStringLiteralTo(absl::string_view str, + std::string* dest) { + dest->push_back('"'); + EscapeStringInternal(dest, str.begin(), str.end(), true, '"'); + dest->push_back('"'); } std::string FormatBytesLiteral(absl::string_view str) { - absl::string_view quote = - (str.find('"') != str.npos && str.find('\'') == str.npos) ? "'" : "\""; - return absl::StrCat("b", quote, EscapeBytes(str, false, quote[0]), quote); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 3); + FormatBytesLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +std::string FormatBytesLiteral(const absl::Cord& str) { + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 3); + FormatBytesLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatBytesLiteralTo(absl::string_view str, std::string* dest) { + const char quote = + absl::StrContains(str, '"') && !absl::StrContains(str, '\'') ? '\'' : '"'; + dest->push_back('b'); + dest->push_back(quote); + EscapeBytesInternal(dest, str.begin(), str.end(), false, quote); + dest->push_back(quote); +} + +void FormatBytesLiteralTo(const absl::Cord& str, std::string* dest) { + const char quote = (str.Contains("\"") && !str.Contains("'")) ? '\'' : '"'; + dest->push_back('b'); + dest->push_back(quote); + EscapeBytesInternal(dest, str.char_begin(), str.char_end(), false, quote); + dest->push_back(quote); } std::string FormatSingleQuotedBytesLiteral(absl::string_view str) { - return absl::StrCat("b'", EscapeBytes(str, false, '\''), "'"); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 3); + FormatSingleQuotedBytesLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatSingleQuotedBytesLiteralTo(absl::string_view str, + std::string* dest) { + dest->append("b'"); + EscapeBytesInternal(dest, str.begin(), str.end(), false, '\''); + dest->push_back('\''); } std::string FormatDoubleQuotedBytesLiteral(absl::string_view str) { - return absl::StrCat("b\"", EscapeBytes(str, false, '"'), "\""); + std::string dest; + // Worst case size is every byte has to be hex escaped, so 4 char for every + // byte. + dest.reserve(str.size() * 4 + 3); + FormatDoubleQuotedBytesLiteralTo(str, &dest); + dest.shrink_to_fit(); + return dest; +} + +void FormatDoubleQuotedBytesLiteralTo(absl::string_view str, + std::string* dest) { + dest->append("b\""); + EscapeBytesInternal(dest, str.begin(), str.end(), false, '"'); + dest->push_back('"'); } absl::StatusOr ParseIdentifier(absl::string_view str) { diff --git a/internal/strings.h b/internal/strings.h index ae82a14fd..97a972835 100644 --- a/internal/strings.h +++ b/internal/strings.h @@ -61,26 +61,37 @@ absl::StatusOr ParseBytesLiteral(absl::string_view str); // May choose to quote with ' or " to produce nicer output. std::string FormatStringLiteral(absl::string_view str); std::string FormatStringLiteral(const absl::Cord& str); +void FormatStringLiteralTo(absl::string_view str, std::string* dest); +void FormatStringLiteralTo(const absl::Cord& str, std::string* dest); // Return a quoted and escaped CEL string literal for . // Always uses single quotes. std::string FormatSingleQuotedStringLiteral(absl::string_view str); +void FormatSingleQuotedStringLiteralTo(absl::string_view str, + std::string* dest); // Return a quoted and escaped CEL string literal for . // Always uses double quotes. std::string FormatDoubleQuotedStringLiteral(absl::string_view str); +void FormatDoubleQuotedStringLiteralTo(absl::string_view str, + std::string* dest); // Return a quoted and escaped CEL bytes literal for . // Prefixes with b and may choose to quote with ' or " to produce nicer output. std::string FormatBytesLiteral(absl::string_view str); +std::string FormatBytesLiteral(const absl::Cord& str); +void FormatBytesLiteralTo(absl::string_view str, std::string* dest); +void FormatBytesLiteralTo(const absl::Cord& str, std::string* dest); // Return a quoted and escaped CEL bytes literal for . // Prefixes with b and always uses single quotes. std::string FormatSingleQuotedBytesLiteral(absl::string_view str); +void FormatSingleQuotedBytesLiteralTo(absl::string_view str, std::string* dest); // Return a quoted and escaped CEL bytes literal for . // Prefixes with b and always uses double quotes. std::string FormatDoubleQuotedBytesLiteral(absl::string_view str); +void FormatDoubleQuotedBytesLiteralTo(absl::string_view str, std::string* dest); // Parse a CEL identifier. absl::StatusOr ParseIdentifier(absl::string_view str); diff --git a/internal/strings_test.cc b/internal/strings_test.cc index fcdb6d4ec..3d950d260 100644 --- a/internal/strings_test.cc +++ b/internal/strings_test.cc @@ -86,7 +86,7 @@ void TestBytesEscaping(const std::string& unquoted, const std::string& quoted) { } // takes a byte literal of the form b'...', b'''...''' -void TestBytesLiteral(const std::string& quoted) { +void TestBytesLiteralString(const std::string& quoted) { // Parse the literal. ASSERT_OK_AND_ASSIGN(auto unquoted, ParseBytesLiteral(quoted)); @@ -103,6 +103,30 @@ void TestBytesLiteral(const std::string& quoted) { TestBytesEscaping(unquoted, quoted); } +// takes a byte literal of the form b'...', b'''...''' +void TestBytesLiteralCord(const std::string& quoted) { + // Parse the literal. + ASSERT_OK_AND_ASSIGN(auto unquoted, ParseBytesLiteral(quoted)); + + // Take the parsed literal and turn it back to a literal. + std::string requoted = FormatBytesLiteral(absl::Cord(unquoted)); + // Parse it again. + ASSERT_OK_AND_ASSIGN(auto unquoted2, ParseBytesLiteral(requoted)); + // Test the parsed literal forms for equality, not the unparsed forms. + // This is because the unparsed forms can have different representations for + // the same data, i.e., \000 and \x00. + EXPECT_EQ(unquoted, unquoted2) + << "unquoted : " << unquoted << "\nunquoted2: " << unquoted2; + + TestBytesEscaping(unquoted, quoted); +} + +// takes a byte literal of the form b'...', b'''...''' +void TestBytesLiteral(const std::string& quoted) { + TestBytesLiteralString(quoted); + TestBytesLiteralCord(quoted); +} + // takes a raw byte literal of the form rb'...', br'...', rb'''...''' // or br'''...'''. is the expected parsed form of . void TestQuotedRawBytesLiteral(const std::string& unquoted, From e08ab654aa2d66bb3c6bd8d35abc8955bf8166a7 Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 6 Oct 2026 13:07:07 -0700 Subject: [PATCH 266/350] Remove `cel::Data`, `cel::ReferenceCount`, and related internal primitives PiperOrigin-RevId: 994577785 --- common/BUILD | 30 -- common/allocator.h | 10 +- common/data.h | 120 ------- common/data_test.cc | 67 ---- common/internal/BUILD | 34 -- common/internal/metadata.h | 41 --- common/internal/reference_count.cc | 118 ------- common/internal/reference_count.h | 406 ------------------------ common/internal/reference_count_test.cc | 162 ---------- common/memory.h | 6 +- common/memory_testing.h | 2 - common/reference_count.h | 26 -- 12 files changed, 4 insertions(+), 1018 deletions(-) delete mode 100644 common/data.h delete mode 100644 common/data_test.cc delete mode 100644 common/internal/metadata.h delete mode 100644 common/internal/reference_count.cc delete mode 100644 common/internal/reference_count.h delete mode 100644 common/internal/reference_count_test.cc delete mode 100644 common/reference_count.h diff --git a/common/BUILD b/common/BUILD index a41f0cb1a..14e3b6c0d 100644 --- a/common/BUILD +++ b/common/BUILD @@ -894,18 +894,11 @@ cc_library( ], ) -cc_library( - name = "reference_count", - hdrs = ["reference_count.h"], - deps = ["//common/internal:reference_count"], -) - cc_library( name = "allocator", hdrs = ["allocator.h"], deps = [ ":arena", - ":data", "//internal:new", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", @@ -927,29 +920,6 @@ cc_test( ], ) -cc_library( - name = "data", - hdrs = ["data.h"], - deps = [ - "//common/internal:metadata", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/log:absl_check", - "@com_google_protobuf//:protobuf", - ], -) - -cc_test( - name = "data_test", - srcs = ["data_test.cc"], - deps = [ - ":data", - "//common/internal:reference_count", - "//internal:testing", - "@com_google_absl//absl/base:nullability", - "@com_google_protobuf//:protobuf", - ], -) - cc_library( name = "optional_ref", hdrs = ["optional_ref.h"], diff --git a/common/allocator.h b/common/allocator.h index 81d56b096..a6453d3f9 100644 --- a/common/allocator.h +++ b/common/allocator.h @@ -27,7 +27,6 @@ #include "absl/log/die_if_null.h" #include "absl/numeric/bits.h" #include "common/arena.h" -#include "common/data.h" #include "internal/new.h" #include "google/protobuf/arena.h" @@ -291,9 +290,6 @@ class ArenaAllocator { using U = std::remove_const_t; U* object; if constexpr (google::protobuf::Arena::is_arena_constructable::value) { - // Classes derived from `cel::Data` are manually allocated and constructed - // as those class support determining whether the destructor is skippable - // at runtime. object = google::protobuf::Arena::Create(arena(), std::forward(args)...); } else { if constexpr (ArenaTraits<>::constructible()) { @@ -309,8 +305,7 @@ class ArenaAllocator { } } } - if constexpr (google::protobuf::Arena::is_arena_constructable::value || - std::is_base_of_v) { + if constexpr (google::protobuf::Arena::is_arena_constructable::value) { ABSL_DCHECK_EQ(object->GetArena(), arena()); } return object; @@ -322,8 +317,7 @@ class ArenaAllocator { void delete_object(T* p) noexcept { using U = std::remove_const_t; ABSL_DCHECK(p != nullptr); - if constexpr (google::protobuf::Arena::is_arena_constructable::value || - std::is_base_of_v) { + if constexpr (google::protobuf::Arena::is_arena_constructable::value) { ABSL_DCHECK_EQ(p->GetArena(), arena()); } } diff --git a/common/data.h b/common/data.h deleted file mode 100644 index cefc21fa4..000000000 --- a/common/data.h +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_DATA_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_DATA_H_ - -#include -#include - -#include "absl/base/nullability.h" -#include "absl/log/absl_check.h" -#include "common/internal/metadata.h" -#include "google/protobuf/arena.h" - -namespace cel { - -class Data; -template -struct Ownable; -template -struct Borrowable; - -namespace common_internal { - -class ReferenceCount; - -void SetDataReferenceCount(const Data* absl_nonnull data, - const ReferenceCount* absl_nonnull refcount); - -const ReferenceCount* absl_nullable GetDataReferenceCount( - const Data* absl_nonnull data); - -} // namespace common_internal - -// `Data` is one of the base classes of objects that can be managed by -// `MemoryManager`, the other is `google::protobuf::MessageLite`. -class Data { - public: - Data(const Data&) = default; - Data(Data&&) = default; - ~Data() = default; - Data& operator=(const Data&) = default; - Data& operator=(Data&&) = default; - - google::protobuf::Arena* absl_nullable GetArena() const { - return (owner_ & kOwnerBits) == kOwnerArenaBit - ? reinterpret_cast(owner_ & kOwnerPointerMask) - : nullptr; - } - - protected: - // At this point, the reference count has not been created. So we create it - // unowned and set the reference count after. In theory we could create the - // reference count ahead of time and then update it with the data it has to - // delete, but that is a bit counter intuitive. Doing it this way is also - // similar to how std::enable_shared_from_this works. - Data() = default; - - Data(std::nullptr_t) = delete; - - explicit Data(google::protobuf::Arena* absl_nullable arena) - : owner_(reinterpret_cast(arena) | - (arena != nullptr ? kOwnerArenaBit : kOwnerNone)) {} - - private: - static constexpr uintptr_t kOwnerNone = common_internal::kMetadataOwnerNone; - static constexpr uintptr_t kOwnerReferenceCountBit = - common_internal::kMetadataOwnerReferenceCountBit; - static constexpr uintptr_t kOwnerArenaBit = - common_internal::kMetadataOwnerArenaBit; - static constexpr uintptr_t kOwnerBits = common_internal::kMetadataOwnerBits; - static constexpr uintptr_t kOwnerPointerMask = - common_internal::kMetadataOwnerPointerMask; - - friend void common_internal::SetDataReferenceCount( - const Data* absl_nonnull data, - const common_internal::ReferenceCount* absl_nonnull refcount); - friend const common_internal::ReferenceCount* absl_nullable - common_internal::GetDataReferenceCount(const Data* absl_nonnull data); - template - friend struct Ownable; - template - friend struct Borrowable; - - mutable uintptr_t owner_ = kOwnerNone; -}; - -namespace common_internal { - -inline void SetDataReferenceCount(const Data* absl_nonnull data, - const ReferenceCount* absl_nonnull refcount) { - ABSL_DCHECK_EQ(data->owner_, Data::kOwnerNone); - data->owner_ = - reinterpret_cast(refcount) | Data::kOwnerReferenceCountBit; -} - -inline const ReferenceCount* absl_nullable GetDataReferenceCount( - const Data* absl_nonnull data) { - return (data->owner_ & Data::kOwnerBits) == Data::kOwnerReferenceCountBit - ? reinterpret_cast(data->owner_ & - Data::kOwnerPointerMask) - : nullptr; -} - -} // namespace common_internal - -} // namespace cel - -#endif // THIRD_PARTY_CEL_CPP_COMMON_DATA_H_ diff --git a/common/data_test.cc b/common/data_test.cc deleted file mode 100644 index a6b2a788f..000000000 --- a/common/data_test.cc +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This header contains primitives for reference counting, roughly equivalent to -// the primitives used to implement `std::shared_ptr`. These primitives should -// not be used directly in most cases, instead `cel::ManagedMemory` should be -// used instead. - -#include "common/data.h" - -#include "absl/base/nullability.h" -#include "common/internal/reference_count.h" -#include "internal/testing.h" -#include "google/protobuf/arena.h" - -namespace cel { -namespace { - -using ::testing::IsNull; - -class DataTest final : public Data { - public: - DataTest() noexcept : Data() {} - - explicit DataTest(google::protobuf::Arena* absl_nullable arena) noexcept - : Data(arena) {} -}; - -class DataReferenceCount final : public common_internal::ReferenceCounted { - public: - explicit DataReferenceCount(const Data* data) : data_(data) {} - - private: - void Finalize() noexcept override { delete data_; } - - const Data* data_; -}; - -TEST(Data, Arena) { - google::protobuf::Arena arena; - DataTest data(&arena); - EXPECT_EQ(data.GetArena(), &arena); - EXPECT_THAT(common_internal::GetDataReferenceCount(&data), IsNull()); -} - -TEST(Data, ReferenceCount) { - auto* data = new DataTest(); - EXPECT_THAT(data->GetArena(), IsNull()); - auto* refcount = new DataReferenceCount(data); - common_internal::SetDataReferenceCount(data, refcount); - EXPECT_EQ(common_internal::GetDataReferenceCount(data), refcount); - common_internal::StrongUnref(refcount); -} - -} // namespace -} // namespace cel diff --git a/common/internal/BUILD b/common/internal/BUILD index eebf66219..0b6f6238c 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -28,40 +28,6 @@ cc_library( ], ) -cc_library( - name = "reference_count", - srcs = ["reference_count.cc"], - hdrs = ["reference_count.h"], - deps = [ - "//common:data", - "//internal:new", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/strings:string_view", - "@com_google_protobuf//:protobuf", - ], -) - -cc_test( - name = "reference_count_test", - srcs = ["reference_count_test.cc"], - deps = [ - ":reference_count", - "//common:data", - "//internal:testing", - "@com_google_absl//absl/base:nullability", - "@com_google_protobuf//:protobuf", - "@com_google_protobuf//:struct_cc_proto", - ], -) - -cc_library( - name = "metadata", - hdrs = ["metadata.h"], - deps = ["@com_google_protobuf//:protobuf"], -) - cc_library( name = "byte_string", srcs = ["byte_string.cc"], diff --git a/common/internal/metadata.h b/common/internal/metadata.h deleted file mode 100644 index 5d2fa8322..000000000 --- a/common/internal/metadata.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_METADATA_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_METADATA_H_ - -#include - -#include "google/protobuf/arena.h" - -namespace cel::common_internal { - -// `google::protobuf::Arena` has a minimum alignment of 8. `ReferenceCount` has a minimum -// alignment that is guaranteed to be greater than or equal to `google::protobuf::Arena`. -inline constexpr uintptr_t kMetadataOwnerNone = 0; -inline constexpr uintptr_t kMetadataOwnerReferenceCountBit = uintptr_t{1} << 0; -inline constexpr uintptr_t kMetadataOwnerArenaBit = uintptr_t{1} << 1; -inline constexpr uintptr_t kMetadataOwnerBits = alignof(google::protobuf::Arena) - 1; -inline constexpr uintptr_t kMetadataOwnerPointerMask = ~kMetadataOwnerBits; - -// Ensure kMetadataOwnerBits encompasses kMetadataOwnerReferenceCountBit and -// kMetadataOwnerArenaBit. -static_assert((kMetadataOwnerBits | kMetadataOwnerReferenceCountBit) == - kMetadataOwnerBits); -static_assert((kMetadataOwnerBits | kMetadataOwnerArenaBit) == - kMetadataOwnerBits); - -} // namespace cel::common_internal - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_METADATA_H_ diff --git a/common/internal/reference_count.cc b/common/internal/reference_count.cc deleted file mode 100644 index c954c685e..000000000 --- a/common/internal/reference_count.cc +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/internal/reference_count.h" - -#include -#include -#include -#include -#include -#include - -#include "absl/base/nullability.h" -#include "absl/log/absl_check.h" -#include "absl/strings/string_view.h" -#include "common/data.h" -#include "internal/new.h" -#include "google/protobuf/message_lite.h" - -namespace cel::common_internal { - -template class DeletingReferenceCount; - -namespace { - -class ReferenceCountedStdString final : public ReferenceCounted { - public: - static std::pair New( - std::string&& string) { - const auto* const refcount = - new ReferenceCountedStdString(std::move(string)); - const auto* const refcount_string = std::launder( - reinterpret_cast(&refcount->string_[0])); - return std::pair{static_cast(refcount), - absl::string_view(*refcount_string)}; - } - - explicit ReferenceCountedStdString(std::string&& string) { - (::new (static_cast(&string_[0])) std::string(std::move(string))) - ->shrink_to_fit(); - } - - private: - void Finalize() noexcept override { - std::destroy_at(std::launder(reinterpret_cast(&string_[0]))); - } - - alignas(std::string) char string_[sizeof(std::string)]; -}; - -class ReferenceCountedString final : public ReferenceCounted { - public: - static std::pair New( - absl::string_view string) { - const auto* const refcount = - ::new (internal::New(Overhead() + string.size())) - ReferenceCountedString(string); - return std::pair{static_cast(refcount), - absl::string_view(refcount->data_, refcount->size_)}; - } - - private: -// ReferenceCountedString is non-standard-layout due to having virtual functions -// from a base class. This causes compilers to warn about the use of offsetof(), -// but it still works here, so silence the warning and proceed. -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Winvalid-offsetof" -#endif - - static size_t Overhead() { return offsetof(ReferenceCountedString, data_); } - -#if defined(__GNUC__) || defined(__clang__) -#pragma GCC diagnostic pop -#endif - - explicit ReferenceCountedString(absl::string_view string) - : size_(string.size()) { - std::memcpy(data_, string.data(), size_); - } - - void Delete() noexcept override { - void* const that = this; - const auto size = size_; - std::destroy_at(this); - internal::SizedDelete(that, Overhead() + size); - } - - const size_t size_; - char data_[]; -}; - -} // namespace - -std::pair -MakeReferenceCountedString(absl::string_view value) { - ABSL_DCHECK(!value.empty()); - return ReferenceCountedString::New(value); -} - -std::pair -MakeReferenceCountedString(std::string&& value) { - ABSL_DCHECK(!value.empty()); - return ReferenceCountedStdString::New(std::move(value)); -} - -} // namespace cel::common_internal diff --git a/common/internal/reference_count.h b/common/internal/reference_count.h deleted file mode 100644 index 9c7fb5371..000000000 --- a/common/internal/reference_count.h +++ /dev/null @@ -1,406 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This header contains primitives for reference counting, roughly equivalent to -// the primitives used to implement `std::shared_ptr`. These primitives should -// not be used directly in most cases, instead `cel::Shared` should be -// used instead. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_REFERENCE_COUNT_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_REFERENCE_COUNT_H_ - -#include -#include -#include -#include -#include -#include - -#include "absl/base/attributes.h" -#include "absl/base/nullability.h" -#include "absl/base/optimization.h" -#include "absl/log/absl_check.h" -#include "absl/strings/string_view.h" -#include "common/data.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/message_lite.h" - -namespace cel::common_internal { - -struct AdoptRef final { - explicit AdoptRef() = default; -}; - -inline constexpr AdoptRef kAdoptRef{}; - -class ReferenceCount; -struct ReferenceCountFromThis; - -void SetReferenceCountForThat(ReferenceCountFromThis& that, - ReferenceCount* absl_nullable refcount); - -ReferenceCount* absl_nullable GetReferenceCountForThat( - const ReferenceCountFromThis& that); - -// `ReferenceCountFromThis` is similar to `std::enable_shared_from_this`. It -// allows the derived object to inspect its own reference count. It should not -// be used directly, but should be used through -// `cel::EnableManagedMemoryFromThis`. -struct ReferenceCountFromThis { - private: - friend void SetReferenceCountForThat(ReferenceCountFromThis& that, - ReferenceCount* absl_nullable refcount); - friend ReferenceCount* absl_nullable GetReferenceCountForThat( - const ReferenceCountFromThis& that); - - static constexpr uintptr_t kNullPtr = uintptr_t{0}; - static constexpr uintptr_t kSentinelPtr = ~kNullPtr; - - void* absl_nullable refcount = reinterpret_cast(kSentinelPtr); -}; - -inline void SetReferenceCountForThat(ReferenceCountFromThis& that, - ReferenceCount* absl_nullable refcount) { - ABSL_DCHECK_EQ(that.refcount, - reinterpret_cast(ReferenceCountFromThis::kSentinelPtr)); - that.refcount = static_cast(refcount); -} - -inline ReferenceCount* absl_nullable GetReferenceCountForThat( - const ReferenceCountFromThis& that) { - ABSL_DCHECK_NE(that.refcount, - reinterpret_cast(ReferenceCountFromThis::kSentinelPtr)); - return static_cast(that.refcount); -} - -void StrongRef(const ReferenceCount& refcount) noexcept; - -void StrongRef(const ReferenceCount* absl_nullable refcount) noexcept; - -void StrongUnref(const ReferenceCount& refcount) noexcept; - -void StrongUnref(const ReferenceCount* absl_nullable refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool StrengthenRef(const ReferenceCount& refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool StrengthenRef(const ReferenceCount* absl_nullable refcount) noexcept; - -void WeakRef(const ReferenceCount& refcount) noexcept; - -void WeakRef(const ReferenceCount* absl_nullable refcount) noexcept; - -void WeakUnref(const ReferenceCount& refcount) noexcept; - -void WeakUnref(const ReferenceCount* absl_nullable refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool IsUniqueRef(const ReferenceCount& refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool IsUniqueRef(const ReferenceCount* absl_nullable refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool IsExpiredRef(const ReferenceCount& refcount) noexcept; - -ABSL_MUST_USE_RESULT -bool IsExpiredRef(const ReferenceCount* absl_nullable refcount) noexcept; - -// `ReferenceCount` is similar to the control block used by `std::shared_ptr`. -// It is not meant to be interacted with directly in most cases, instead -// `cel::Shared` should be used. -class alignas(8) ReferenceCount { - public: - ReferenceCount() = default; - - ReferenceCount(const ReferenceCount&) = delete; - ReferenceCount(ReferenceCount&&) = delete; - ReferenceCount& operator=(const ReferenceCount&) = delete; - ReferenceCount& operator=(ReferenceCount&&) = delete; - - virtual ~ReferenceCount() = default; - - private: - friend void StrongRef(const ReferenceCount& refcount) noexcept; - friend void StrongUnref(const ReferenceCount& refcount) noexcept; - friend bool StrengthenRef(const ReferenceCount& refcount) noexcept; - friend void WeakRef(const ReferenceCount& refcount) noexcept; - friend void WeakUnref(const ReferenceCount& refcount) noexcept; - friend bool IsUniqueRef(const ReferenceCount& refcount) noexcept; - friend bool IsExpiredRef(const ReferenceCount& refcount) noexcept; - - virtual void Finalize() noexcept = 0; - - virtual void Delete() noexcept = 0; - - mutable std::atomic strong_refcount_ = 1; - mutable std::atomic weak_refcount_ = 1; -}; - -// ReferenceCount and its derivations must be at least as aligned as -// google::protobuf::Arena. This is a requirement for the pointer tagging defined in -// common/internal/metadata.h. -static_assert(alignof(ReferenceCount) >= alignof(google::protobuf::Arena)); - -// `ReferenceCounted` is a base class for classes which should be reference -// counted. It provides default implementations for `Finalize()` and `Delete()`. -class ReferenceCounted : public ReferenceCount { - private: - void Finalize() noexcept override {} - - void Delete() noexcept override { delete this; } -}; - -// `EmplacedReferenceCount` adapts `T` to make it reference countable, by -// storing `T` inside the reference count. This only works when `T` has not yet -// been allocated. -template -class EmplacedReferenceCount final : public ReferenceCounted { - public: - static_assert(std::is_destructible_v, "T must be destructible"); - static_assert(!std::is_reference_v, "T must not be a reference"); - static_assert(!std::is_volatile_v, "T must not be volatile qualified"); - static_assert(!std::is_const_v, "T must not be const qualified"); - static_assert(!std::is_array_v, "T must not be an array"); - - template - explicit EmplacedReferenceCount(T*& value, Args&&... args) noexcept( - std::is_nothrow_constructible_v) { - value = - ::new (static_cast(&value_[0])) T(std::forward(args)...); - } - - private: - void Finalize() noexcept override { - std::destroy_at(std::launder(reinterpret_cast(&value_[0]))); - } - - // We store the instance of `T` in a char buffer and use placement new and - // direct calls to the destructor. The reason for this is `Finalize()` is - // called when the strong reference count hits 0. This allows us to destroy - // our instance of `T` once we are no longer strongly reachable and deallocate - // the memory once we are no longer weakly reachable. - alignas(T) char value_[sizeof(T)]; -}; - -// `DeletingReferenceCount` adapts `T` to make it reference countable, by taking -// ownership of `T` and deleting it. This only works when `T` has already been -// allocated and is to expensive to move or copy. -template -class DeletingReferenceCount final : public ReferenceCounted { - public: - explicit DeletingReferenceCount(const T* absl_nonnull to_delete) noexcept - : to_delete_(to_delete) {} - - private: - void Finalize() noexcept override { delete to_delete_; } - - const T* absl_nonnull const to_delete_; -}; - -extern template class DeletingReferenceCount; - -template -const ReferenceCount* absl_nonnull MakeDeletingReferenceCount( - const T* absl_nonnull to_delete) { - if constexpr (google::protobuf::Arena::is_arena_constructable::value) { - ABSL_DCHECK_EQ(to_delete->GetArena(), nullptr); - } - if constexpr (std::is_base_of_v) { - return new DeletingReferenceCount(to_delete); - } else { - auto* refcount = new DeletingReferenceCount(to_delete); - if constexpr (std::is_base_of_v) { - common_internal::SetDataReferenceCount(to_delete, refcount); - } - return refcount; - } -} - -template -std::pair -MakeEmplacedReferenceCount(Args&&... args) { - using U = std::remove_const_t; - U* pointer; - auto* const refcount = - new EmplacedReferenceCount(pointer, std::forward(args)...); - if constexpr (google::protobuf::Arena::is_arena_constructable::value) { - ABSL_DCHECK_EQ(pointer->GetArena(), nullptr); - } - if constexpr (std::is_base_of_v) { - common_internal::SetDataReferenceCount(pointer, refcount); - } - return std::pair{static_cast(pointer), - static_cast(refcount)}; -} - -template -class InlinedReferenceCount final : public ReferenceCounted { - public: - template - explicit InlinedReferenceCount(std::in_place_t, Args&&... args) - : ReferenceCounted() { - ::new (static_cast(value())) T(std::forward(args)...); - } - - ABSL_ATTRIBUTE_ALWAYS_INLINE T* absl_nonnull value() { - return reinterpret_cast(&value_[0]); - } - - ABSL_ATTRIBUTE_ALWAYS_INLINE const T* absl_nonnull value() const { - return reinterpret_cast(&value_[0]); - } - - private: - void Finalize() noexcept override { value()->~T(); } - - // We store the instance of `T` in a char buffer and use placement new and - // direct calls to the destructor. The reason for this is `Finalize()` is - // called when the strong reference count hits 0. This allows us to destroy - // our instance of `T` once we are no longer strongly reachable and deallocate - // the memory once we are no longer weakly reachable. - alignas(T) char value_[sizeof(T)]; -}; - -template -std::pair MakeReferenceCount( - Args&&... args) { - using U = std::remove_const_t; - auto* const refcount = - new InlinedReferenceCount(std::in_place, std::forward(args)...); - auto* const pointer = refcount->value(); - if constexpr (std::is_base_of_v) { - SetReferenceCountForThat(*pointer, refcount); - } - return std::make_pair(static_cast(pointer), - static_cast(refcount)); -} - -inline void StrongRef(const ReferenceCount& refcount) noexcept { - const auto count = - refcount.strong_refcount_.fetch_add(1, std::memory_order_relaxed); - ABSL_DCHECK_GT(count, 0); -} - -inline void StrongRef(const ReferenceCount* absl_nullable refcount) noexcept { - if (refcount != nullptr) { - StrongRef(*refcount); - } -} - -inline void StrongUnref(const ReferenceCount& refcount) noexcept { - const auto count = - refcount.strong_refcount_.fetch_sub(1, std::memory_order_acq_rel); - ABSL_DCHECK_GT(count, 0); - ABSL_ASSUME(count > 0); - if (ABSL_PREDICT_FALSE(count == 1)) { - const_cast(refcount).Finalize(); - WeakUnref(refcount); - } -} - -inline void StrongUnref(const ReferenceCount* absl_nullable refcount) noexcept { - if (refcount != nullptr) { - StrongUnref(*refcount); - } -} - -ABSL_MUST_USE_RESULT -inline bool StrengthenRef(const ReferenceCount& refcount) noexcept { - auto count = refcount.strong_refcount_.load(std::memory_order_relaxed); - while (true) { - ABSL_DCHECK_GE(count, 0); - ABSL_ASSUME(count >= 0); - if (count == 0) { - return false; - } - if (refcount.strong_refcount_.compare_exchange_weak( - count, count + 1, std::memory_order_release, - std::memory_order_relaxed)) { - return true; - } - } -} - -ABSL_MUST_USE_RESULT -inline bool StrengthenRef( - const ReferenceCount* absl_nullable refcount) noexcept { - return refcount != nullptr ? StrengthenRef(*refcount) : false; -} - -inline void WeakRef(const ReferenceCount& refcount) noexcept { - const auto count = - refcount.weak_refcount_.fetch_add(1, std::memory_order_relaxed); - ABSL_DCHECK_GT(count, 0); -} - -inline void WeakRef(const ReferenceCount* absl_nullable refcount) noexcept { - if (refcount != nullptr) { - WeakRef(*refcount); - } -} - -inline void WeakUnref(const ReferenceCount& refcount) noexcept { - const auto count = - refcount.weak_refcount_.fetch_sub(1, std::memory_order_acq_rel); - ABSL_DCHECK_GT(count, 0); - ABSL_ASSUME(count > 0); - if (ABSL_PREDICT_FALSE(count == 1)) { - const_cast(refcount).Delete(); - } -} - -inline void WeakUnref(const ReferenceCount* absl_nullable refcount) noexcept { - if (refcount != nullptr) { - WeakUnref(*refcount); - } -} - -ABSL_MUST_USE_RESULT -inline bool IsUniqueRef(const ReferenceCount& refcount) noexcept { - const auto count = refcount.strong_refcount_.load(std::memory_order_acquire); - ABSL_DCHECK_GT(count, 0); - ABSL_ASSUME(count > 0); - return count == 1; -} - -ABSL_MUST_USE_RESULT -inline bool IsUniqueRef(const ReferenceCount* absl_nullable refcount) noexcept { - return refcount != nullptr ? IsUniqueRef(*refcount) : false; -} - -ABSL_MUST_USE_RESULT -inline bool IsExpiredRef(const ReferenceCount& refcount) noexcept { - const auto count = refcount.strong_refcount_.load(std::memory_order_acquire); - ABSL_DCHECK_GE(count, 0); - ABSL_ASSUME(count >= 0); - return count == 0; -} - -ABSL_MUST_USE_RESULT -inline bool IsExpiredRef( - const ReferenceCount* absl_nullable refcount) noexcept { - return refcount != nullptr ? IsExpiredRef(*refcount) : false; -} - -std::pair -MakeReferenceCountedString(absl::string_view value); - -std::pair -MakeReferenceCountedString(std::string&& value); - -} // namespace cel::common_internal - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_REFERENCE_COUNT_H_ diff --git a/common/internal/reference_count_test.cc b/common/internal/reference_count_test.cc deleted file mode 100644 index af36fa9a5..000000000 --- a/common/internal/reference_count_test.cc +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/internal/reference_count.h" - -#include - -#include "google/protobuf/struct.pb.h" -#include "absl/base/nullability.h" -#include "common/data.h" -#include "internal/testing.h" -#include "google/protobuf/arena.h" -#include "google/protobuf/message_lite.h" - -namespace cel::common_internal { -namespace { - -using ::testing::NotNull; -using ::testing::WhenDynamicCastTo; - -class Object : public virtual ReferenceCountFromThis { - public: - explicit Object(bool& destructed) : destructed_(destructed) {} - - ~Object() { destructed_ = true; } - - private: - bool& destructed_; -}; - -class Subobject : public Object, public virtual ReferenceCountFromThis { - public: - using Object::Object; -}; - -TEST(ReferenceCount, Strong) { - bool destructed = false; - Object* object; - ReferenceCount* refcount; - std::tie(object, refcount) = MakeReferenceCount(destructed); - EXPECT_EQ(GetReferenceCountForThat(*object), refcount); - EXPECT_EQ(GetReferenceCountForThat(*static_cast(object)), - refcount); - StrongRef(refcount); - StrongUnref(refcount); - EXPECT_TRUE(IsUniqueRef(refcount)); - EXPECT_FALSE(IsExpiredRef(refcount)); - EXPECT_FALSE(destructed); - StrongUnref(refcount); - EXPECT_TRUE(destructed); -} - -TEST(ReferenceCount, Weak) { - bool destructed = false; - Object* object; - ReferenceCount* refcount; - std::tie(object, refcount) = MakeReferenceCount(destructed); - EXPECT_EQ(GetReferenceCountForThat(*object), refcount); - EXPECT_EQ(GetReferenceCountForThat(*static_cast(object)), - refcount); - WeakRef(refcount); - ASSERT_TRUE(StrengthenRef(refcount)); - StrongUnref(refcount); - EXPECT_TRUE(IsUniqueRef(refcount)); - EXPECT_FALSE(IsExpiredRef(refcount)); - EXPECT_FALSE(destructed); - StrongUnref(refcount); - EXPECT_TRUE(destructed); - EXPECT_TRUE(IsExpiredRef(refcount)); - ASSERT_FALSE(StrengthenRef(refcount)); - WeakUnref(refcount); -} - -class DataObject final : public Data { - public: - DataObject() noexcept : Data() {} - - explicit DataObject(google::protobuf::Arena* absl_nullable arena) noexcept - : Data(arena) {} - - char member_[17]; -}; - -struct OtherObject final { - char data[17]; -}; - -TEST(DeletingReferenceCount, Data) { - auto* data = new DataObject(); - const auto* refcount = MakeDeletingReferenceCount(data); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>(NotNull())); - EXPECT_EQ(common_internal::GetDataReferenceCount(data), refcount); - StrongUnref(refcount); -} - -TEST(DeletingReferenceCount, MessageLite) { - auto* message_lite = new google::protobuf::Value(); - const auto* refcount = MakeDeletingReferenceCount(message_lite); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>( - NotNull())); - StrongUnref(refcount); -} - -TEST(DeletingReferenceCount, Other) { - auto* other = new OtherObject(); - const auto* refcount = MakeDeletingReferenceCount(other); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>(NotNull())); - StrongUnref(refcount); -} - -TEST(EmplacedReferenceCount, Data) { - Data* data; - const ReferenceCount* refcount; - std::tie(data, refcount) = MakeEmplacedReferenceCount(); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>(NotNull())); - EXPECT_EQ(common_internal::GetDataReferenceCount(data), refcount); - StrongUnref(refcount); -} - -TEST(EmplacedReferenceCount, MessageLite) { - google::protobuf::Value* message_lite; - const ReferenceCount* refcount; - std::tie(message_lite, refcount) = - MakeEmplacedReferenceCount(); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>( - NotNull())); - StrongUnref(refcount); -} - -TEST(EmplacedReferenceCount, Other) { - OtherObject* other; - const ReferenceCount* refcount; - std::tie(other, refcount) = MakeEmplacedReferenceCount(); - EXPECT_THAT( - refcount, - WhenDynamicCastTo*>(NotNull())); - StrongUnref(refcount); -} - -} // namespace -} // namespace cel::common_internal diff --git a/common/memory.h b/common/memory.h index a9ae46023..684e708f8 100644 --- a/common/memory.h +++ b/common/memory.h @@ -49,12 +49,10 @@ enum class MemoryManagement { std::ostream& operator<<(std::ostream& out, MemoryManagement memory_management); class MemoryManager; -class ReferenceCountingMemoryManager; -class PoolingMemoryManager; // `ReferenceCountingMemoryManager` is a `MemoryManager` which employs automatic // memory management through reference counting. -class ReferenceCountingMemoryManager final { +class ABSL_DEPRECATED("Do not use") ReferenceCountingMemoryManager final { public: ReferenceCountingMemoryManager(const ReferenceCountingMemoryManager&) = delete; @@ -76,7 +74,7 @@ class ReferenceCountingMemoryManager final { // `PoolingMemoryManager` is a `MemoryManager` which employs automatic // memory management through memory pooling. -class PoolingMemoryManager final { +class ABSL_DEPRECATED("Do not use") PoolingMemoryManager final { public: PoolingMemoryManager(const PoolingMemoryManager&) = delete; PoolingMemoryManager(PoolingMemoryManager&&) = delete; diff --git a/common/memory_testing.h b/common/memory_testing.h index 37244dd8f..7c06de298 100644 --- a/common/memory_testing.h +++ b/common/memory_testing.h @@ -39,8 +39,6 @@ class ThreadCompatibleMemoryTest MemoryManagerRef memory_manager() { switch (memory_management()) { case MemoryManagement::kReferenceCounting: - return MemoryManager::ReferenceCounting(); - break; case MemoryManagement::kPooling: if (!arena_) { arena_.emplace(); diff --git a/common/reference_count.h b/common/reference_count.h deleted file mode 100644 index 0a07670bd..000000000 --- a/common/reference_count.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_REFERENCE_COUNT_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_REFERENCE_COUNT_H_ - -#include "common/internal/reference_count.h" - -namespace cel { - -using ReferenceCount = common_internal::ReferenceCount; - -} // namespace cel - -#endif // THIRD_PARTY_CEL_CPP_COMMON_REFERENCE_COUNT_H_ From 973d7d3dc8a4c0522fa73f9ae473b6472d281639 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 6 Oct 2026 14:50:49 -0700 Subject: [PATCH 267/350] Remove unnecessary StatusOr wrapping in function step implementation. PiperOrigin-RevId: 994642974 --- eval/eval/function_step.cc | 39 +++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index fe06b8eca..91a9fa87f 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -157,11 +157,13 @@ inline absl::StatusOr Invoke( context.set_embedder_context(frame.embedder_context()); } - CEL_ASSIGN_OR_RETURN(Value result, - overload.implementation.Invoke(args, context)); + absl::StatusOr result = overload.implementation.Invoke(args, context); + if (!result.ok()) { + return result; + } if (frame.unknown_function_results_enabled() && - IsUnknownFunctionResultError(result)) { + IsUnknownFunctionResultError(*result)) { return frame.attribute_utility().CreateUnknownSet(overload.descriptor, expr_id, args); } @@ -402,18 +404,33 @@ void EvaluateFunctionStep(const Step* step, ExecutionFrame& frame) { } // Derived class resolves to a single function overload or none. - absl::StatusOr matched_function = - step->ResolveFunction(input_args, frame); - if (!matched_function.ok()) { - frame.Abort(std::move(matched_function).status()); - return; + // The extra complexity here is to avoid an unnecessary absl::StatusOr + // wrapping ResolveResult for the common case (static resolution). + ResolveResult resolved_function; + if constexpr (std::is_same_v, + decltype(step->ResolveFunction(input_args, + frame))>) { + absl::StatusOr matched_function = + step->ResolveFunction(input_args, frame); + if (!matched_function.ok()) { + frame.Abort(std::move(matched_function).status()); + return; + } + if (matched_function->has_value()) { + resolved_function.emplace(**matched_function); + } + } else { + ResolveResult matched_function = step->ResolveFunction(input_args, frame); + if (matched_function.has_value()) { + resolved_function.emplace(*matched_function); + } } // Overload found and is allowed to consume the arguments. - if (matched_function->has_value() && - ShouldAcceptOverload((*matched_function)->descriptor, input_args)) { + if (resolved_function.has_value() && + ShouldAcceptOverload(resolved_function->descriptor, input_args)) { absl::StatusOr result = - Invoke(**matched_function, step->expr_id_, input_args, frame); + Invoke(*resolved_function, step->expr_id_, input_args, frame); if (!result.ok()) { frame.Abort(std::move(result).status()); return; From 57533e29a30b71e4b84d923af2519afa16cd98e8 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 6 Oct 2026 15:53:44 -0700 Subject: [PATCH 268/350] Eliminate temporary `std::string` copies in CEL protobuf map and `Struct` lookups. Previously, looking up or checking presence of a string key in a protobuf map or `google.protobuf.Struct` allocated or constructed a temporary `std::string` via `StringValue::NativeString()` (or `std::string(str_key.value())`) before populating a `google::protobuf::MapKey`, even when the key was already backed by a contiguous `absl::string_view` and only a stack scratch buffer was needed for fragmented cord-backed strings. This change uses `StringValue::ToStringView(&scratch)` (and direct `absl::string_view` in `DynamicMap` and `LookupMapValue`) before calling `MapKey::SetStringValue`, avoiding heap allocations and redundant copies for string-keyed map and `google.protobuf.Struct` field accesses across: - `ParsedMapFieldValue::Find` and `ParsedMapFieldValue::Has` - `ProtoStringMapKeyFromValueConverter` - `FieldBackedMapImpl::LookupMapValue` - `DynamicMap::Has` and `DynamicMap::operator[]` (`cel_proto_wrap_util.cc`) - `MapKeyFromQualifier` and `ProtoQualifyState::ApplySelectQualifiers` PiperOrigin-RevId: 994679246 --- common/values/parsed_map_field_value.cc | 5 +++-- common/values/struct_value_builder.cc | 5 +++-- eval/public/containers/internal_field_backed_map_impl.cc | 4 +--- eval/public/structs/cel_proto_wrap_util.cc | 4 ++-- extensions/protobuf/internal/qualify.cc | 4 +--- extensions/select_optimization.cc | 3 ++- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/common/values/parsed_map_field_value.cc b/common/values/parsed_map_field_value.cc index e17bfb4d2..7324286fc 100644 --- a/common/values/parsed_map_field_value.cc +++ b/common/values/parsed_map_field_value.cc @@ -304,8 +304,9 @@ bool ValueToProtoMapKey(const Value& key, } case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { if (auto string_key = key.AsString(); string_key) { - proto_key_scratch = string_key->NativeString(); - proto_key->SetStringValue(proto_key_scratch); + // `google::protobuf::MapKey` does not own the string, which may be stored inline + // in `key` or in `proto_key_scratch`; `key` must not be moved. + proto_key->SetStringValue(string_key->ToStringView(&proto_key_scratch)); return true; } return false; diff --git a/common/values/struct_value_builder.cc b/common/values/struct_value_builder.cc index d1e5b4882..1cb117f46 100644 --- a/common/values/struct_value_builder.cc +++ b/common/values/struct_value_builder.cc @@ -391,8 +391,9 @@ absl::StatusOr> ProtoStringMapKeyFromValueConverter( const Value& value, google::protobuf::MapKey& key, std::string& key_string, google::protobuf::Arena* absl_nonnull arena) { if (auto string_value = value.AsString(); string_value) { - key_string = string_value->NativeString(); - key.SetStringValue(key_string); + // `google::protobuf::MapKey` does not own the string, which may be stored inline in + // `value` or in `key_string`; `value` must not be moved. + key.SetStringValue(string_value->ToStringView(&key_string)); return std::nullopt; } return TypeConversionError(value.GetTypeName(), "string", arena); diff --git a/eval/public/containers/internal_field_backed_map_impl.cc b/eval/public/containers/internal_field_backed_map_impl.cc index 2d5666c58..e4569aa82 100644 --- a/eval/public/containers/internal_field_backed_map_impl.cc +++ b/eval/public/containers/internal_field_backed_map_impl.cc @@ -174,7 +174,6 @@ absl::StatusOr FieldBackedMapImpl::LookupMapValue( return InvalidMapKeyType(key_desc_->cpp_type_name()); } - std::string map_key_string; google::protobuf::MapKey proto_key; switch (key_desc_->cpp_type()) { case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { @@ -199,8 +198,7 @@ absl::StatusOr FieldBackedMapImpl::LookupMapValue( case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { CelValue::StringHolder key_value; key.GetValue(&key_value); - map_key_string.assign(key_value.value().data(), key_value.value().size()); - proto_key.SetStringValue(map_key_string); + proto_key.SetStringValue(key_value.value()); } break; case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { uint64_t key_value; diff --git a/eval/public/structs/cel_proto_wrap_util.cc b/eval/public/structs/cel_proto_wrap_util.cc index 87d05ee3f..08d716d46 100644 --- a/eval/public/structs/cel_proto_wrap_util.cc +++ b/eval/public/structs/cel_proto_wrap_util.cc @@ -111,7 +111,7 @@ class DynamicMap : public CelMap { "Invalid map key type: '", CelValue::TypeName(key.type()), "'")); } - return values_->fields().contains(std::string(str_key.value())); + return values_->fields().contains(str_key.value()); } absl::optional operator[](CelValue key) const override; @@ -692,7 +692,7 @@ absl::optional DynamicMap::operator[](CelValue key) const { CelValue::TypeName(key.type()), "'"))); } - auto it = values_->fields().find(std::string(str_key.value())); + auto it = values_->fields().find(str_key.value()); if (it == values_->fields().end()) { return std::nullopt; } diff --git a/extensions/protobuf/internal/qualify.cc b/extensions/protobuf/internal/qualify.cc index 1d53dac0d..7c1503d1c 100644 --- a/extensions/protobuf/internal/qualify.cc +++ b/extensions/protobuf/internal/qualify.cc @@ -104,7 +104,6 @@ absl::StatusOr> LookupMapValu key_desc->cpp_type_name()); } - std::string proto_key_string; google::protobuf::MapKey proto_key; switch (key_desc->cpp_type()) { case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: @@ -122,8 +121,7 @@ absl::StatusOr> LookupMapValu proto_key.SetInt64Value(*key.GetInt64Key()); break; case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { - proto_key_string = std::string(*key.GetStringKey()); - proto_key.SetStringValue(proto_key_string); + proto_key.SetStringValue(*key.GetStringKey()); } break; case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { uint64_t key_value = *key.GetUint64Key(); diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 5244907e6..8e1a95440 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -270,7 +270,8 @@ absl::StatusOr MapKeyFromQualifier(const AttributeQualifier& qual, case Kind::kBool: return cel::BoolValue(*qual.GetBoolKey()); case Kind::kString: - return StringValue::From(*qual.GetStringKey(), arena); + // Safe as `qual` is valid for the lifetime of the expression/program. + return StringValue::WrapUnsafe(*qual.GetStringKey()); default: return runtime_internal::CreateNoMatchingOverloadError( cel::builtin::kIndex); From 00a1ba3c8f7fc9e72b6e778728438bac3d1338b0 Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 6 Oct 2026 16:14:23 -0700 Subject: [PATCH 269/350] Deprecate recursive planning PiperOrigin-RevId: 994691472 --- eval/public/cel_options.h | 6 ++++++ runtime/runtime_options.h | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index 32474e761..d9ebaab13 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -188,12 +188,18 @@ struct InterpreterOptions { // -1 means unbounded. // 0 means disabled (using a heap-based stack machine instead), which is the // default. + ABSL_DEPRECATED( + "The ability to enable recursive planning is being removed in the " + "near future") int max_recursion_depth = 0; // Enable tracing support for recursively planned programs. // // Unlike the stack machine implementation, supporting tracing can affect // performance whether or not tracing is requested for a given evaluation. + ABSL_DEPRECATED( + "The ability to enable recursive planning is being removed in the " + "near future") bool enable_recursive_tracing = false; // Enable fast implementations for some CEL standard functions. diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index bc6275afb..74ad8ecb7 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -156,12 +156,18 @@ struct RuntimeOptions { // -1 means unbounded. // 0 means disabled (using a heap-based stack machine instead), which is the // default. + ABSL_DEPRECATED( + "The ability to enable recursive planning is being removed in the " + "near future") int max_recursion_depth = 0; // Enable tracing support for recursively planned programs. // // Unlike the stack machine implementation, supporting tracing can affect // performance whether or not tracing is requested for a given evaluation. + ABSL_DEPRECATED( + "The ability to enable recursive planning is being removed in the " + "near future") bool enable_recursive_tracing = false; // Enable fast implementations for some CEL standard functions. From 56b53afecf70c0fef0b244f634d813c394cd43e6 Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 6 Oct 2026 17:40:22 -0700 Subject: [PATCH 270/350] Remove deprecated CEL casting utilities PiperOrigin-RevId: 994736001 --- common/BUILD | 12 - common/casting.h | 75 ------ common/internal/BUILD | 11 - common/internal/casting.h | 237 ------------------ common/legacy_value.cc | 33 ++- common/type_reflector_test.cc | 49 ++-- common/values/custom_list_value.cc | 5 +- common/values/list_value_test.cc | 27 +- common/values/map_value_test.cc | 51 ++-- common/values/null_value_test.cc | 18 -- eval/compiler/BUILD | 3 - .../regex_precompilation_optimization.cc | 12 +- eval/eval/BUILD | 20 -- eval/eval/attribute_utility.cc | 4 +- eval/eval/comprehension_step.cc | 3 - eval/eval/container_access_step.cc | 15 +- eval/eval/create_list_step.cc | 3 - eval/eval/create_list_step_test.cc | 36 ++- eval/eval/create_map_step.cc | 3 - eval/eval/create_struct_step.cc | 3 - eval/eval/function_step.cc | 3 +- eval/eval/ident_step_test.cc | 19 +- eval/eval/logic_step.cc | 13 +- eval/eval/logic_step_test.cc | 3 - eval/eval/optional_or_step.cc | 27 +- eval/eval/optional_or_step_test.cc | 3 - eval/eval/select_step_test.cc | 76 +++--- eval/eval/ternary_step_test.cc | 29 +-- eval/tests/BUILD | 2 - eval/tests/modern_benchmark_test.cc | 92 ++++--- extensions/BUILD | 2 - extensions/math_ext.cc | 7 +- extensions/protobuf/BUILD | 4 - .../protobuf/bind_proto_to_activation_test.cc | 6 +- extensions/protobuf/value_test.cc | 83 +++--- extensions/select_optimization.cc | 8 +- runtime/BUILD | 3 - runtime/bind_proto_to_activation_test.cc | 8 +- runtime/internal/BUILD | 1 - runtime/internal/function_adapter.h | 11 +- runtime/optional_types.cc | 11 +- .../container_membership_functions.cc | 13 +- 42 files changed, 289 insertions(+), 755 deletions(-) delete mode 100644 common/casting.h delete mode 100644 common/internal/casting.h diff --git a/common/BUILD b/common/BUILD index 14e3b6c0d..27ae6ded9 100644 --- a/common/BUILD +++ b/common/BUILD @@ -143,7 +143,6 @@ cc_test( "//internal:testing_descriptor_pool", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", ], ) @@ -463,15 +462,6 @@ cc_test( ], ) -cc_library( - name = "casting", - hdrs = ["casting.h"], - deps = [ - "//common/internal:casting", - "@com_google_absl//absl/base:core_headers", - ], -) - cc_library( name = "json", hdrs = ["json.h"], @@ -757,7 +747,6 @@ cc_library( ":allocator", ":any", ":arena", - ":casting", ":kind", ":memory", ":native_type", @@ -833,7 +822,6 @@ cc_test( "value_test.cc", ], deps = [ - ":casting", ":memory", ":native_type", ":type", diff --git a/common/casting.h b/common/casting.h deleted file mode 100644 index 69074d4d9..000000000 --- a/common/casting.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_CASTING_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_CASTING_H_ - -#include "absl/base/attributes.h" -#include "common/internal/casting.h" - -namespace cel { - -// `InstanceOf(const From&)` determines whether `From` holds or is `To`. -// -// `To` must be a plain non-union class type that is not qualified. -// -// We expose `InstanceOf` this way to avoid ADL. -// -// Example: -// -// if (InstanceOf(superclass)) { -// Cast(superclass).SomeMethod(); -// } -template -ABSL_DEPRECATED("Use Is member functions instead.") -inline constexpr common_internal::InstanceOfImpl InstanceOf{}; - -// `Cast(From)` is a "checked cast". In debug builds an assertion is emitted -// which verifies `From` is an instance-of `To`. In non-debug builds, invalid -// casts are undefined behavior. -// -// We expose `Cast` this way to avoid ADL. -// -// Example: -// -// if (InstanceOf(superclass)) { -// Cast(superclass).SomeMethod(); -// } -template -ABSL_DEPRECATED( - "Use explicit conversion functions instead through static_cast.") -inline constexpr common_internal::CastImpl Cast{}; - -// `As(From)` is a "checking cast". The result is explicitly convertible to -// `bool`, such that it can be used with `if` statements. The result can be -// accessed with `operator*` or `operator->`. The return type should be treated -// as an implementation detail, with no assumptions on the concrete type. You -// should use `auto`. -// -// `As` is analogous to the paradigm `if (InstanceOf(a)) Cast(a)`. -// -// We expose `As` this way to avoid ADL. -// -// Example: -// -// if (auto subclass = As(superclass); subclass) { -// subclass->SomeMethod(); -// } -template -ABSL_DEPRECATED("Use As member functions instead.") -inline constexpr common_internal::AsImpl As{}; - -} // namespace cel - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INSTANCE_OF_H_ diff --git a/common/internal/BUILD b/common/internal/BUILD index 0b6f6238c..571f1ef14 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -17,17 +17,6 @@ load("@rules_cc//cc:cc_test.bzl", "cc_test") package(default_visibility = ["//visibility:public"]) -cc_library( - name = "casting", - hdrs = ["casting.h"], - deps = [ - "//internal:casts", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/meta:type_traits", - "@com_google_absl//absl/types:optional", - ], -) - cc_library( name = "byte_string", srcs = ["byte_string.cc"], diff --git a/common/internal/casting.h b/common/internal/casting.h deleted file mode 100644 index fe7d03279..000000000 --- a/common/internal/casting.h +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// IWYU pragma: private, include "common/casting.h" - -#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_CASTING_H_ -#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_CASTING_H_ - -#include -#include -#include - -#include "absl/base/attributes.h" -#include "absl/meta/type_traits.h" -#include "absl/types/optional.h" -#include "internal/casts.h" - -namespace cel { - -namespace common_internal { - -template -using propagate_const_t = - std::conditional_t>, - std::add_const_t, To>; - -template -using propagate_volatile_t = - std::conditional_t>, - std::add_volatile_t, To>; - -template -using propagate_reference_t = - std::conditional_t, - std::add_lvalue_reference_t, - std::conditional_t, - std::add_rvalue_reference_t, To>>; - -template -using propagate_cvref_t = propagate_reference_t< - propagate_volatile_t, From>, From>; - -} // namespace common_internal - -namespace common_internal { - -// Implementation of `cel::InstanceOf`. -template -struct ABSL_DEPRECATED("Use Is member functions instead.") - InstanceOfImpl final { - static_assert(!std::is_pointer_v, "To must not be a pointer"); - static_assert(!std::is_array_v, "To must not be an array"); - static_assert(!std::is_lvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_rvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_const_v, "To must not be const qualified"); - static_assert(!std::is_volatile_v, "To must not be volatile qualified"); - static_assert(std::is_class_v, "To must be a non-union class"); - - explicit InstanceOfImpl() = default; - - template - ABSL_DEPRECATED("Use Is member functions instead.") - ABSL_MUST_USE_RESULT bool operator()(const From& from) const { - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v, "From must be a non-union class"); - if constexpr (std::is_same_v, To>) { - // Same type. Separate from the next `else if` to work on in-complete - // types. - return true; - } else if constexpr (std::is_polymorphic_v && - std::is_polymorphic_v> && - std::is_base_of_v>) { - // Polymorphic upcast. - return true; - } else if constexpr (!std::is_polymorphic_v && - !std::is_polymorphic_v> && - (std::is_convertible_v || - std::is_convertible_v || - std::is_convertible_v || - std::is_convertible_v)) { - // Implicitly convertible. - return true; - } else { - // Something else. - return from.template Is(); - } - } - - template - ABSL_DEPRECATED("Use Is member functions instead.") - ABSL_MUST_USE_RESULT bool operator()(const From* from) const { - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v, "From must be a non-union class"); - return from != nullptr && (*this)(*from); - } -}; - -// Implementation of `cel::Cast`. -template -struct ABSL_DEPRECATED( - "Use explicit conversion functions instead through static_cast.") - CastImpl final { - static_assert(!std::is_pointer_v, "To must not be a pointer"); - static_assert(!std::is_array_v, "To must not be an array"); - static_assert(!std::is_lvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_rvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_const_v, "To must not be const qualified"); - static_assert(!std::is_volatile_v, "To must not be volatile qualified"); - static_assert(std::is_class_v, "To must be a non-union class"); - - explicit CastImpl() = default; - - template - ABSL_DEPRECATED( - "Use explicit conversion functions instead through static_cast.") - ABSL_MUST_USE_RESULT decltype(auto) - operator()(From&& from) const { - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v>, - "From must be a non-union class"); - if constexpr (std::is_polymorphic_v) { - static_assert(std::is_lvalue_reference_v, - "polymorphic casts are only possible on lvalue references"); - } - if constexpr (std::is_same_v, To>) { - // Same type. Separate from the next `else if` to work on in-complete - // types. - return static_cast>(from); - } else if constexpr (std::is_polymorphic_v && - std::is_polymorphic_v> && - std::is_base_of_v>) { - // Polymorphic upcast. - return static_cast>(from); - } else if constexpr (std::is_polymorphic_v && - std::is_polymorphic_v> && - std::is_base_of_v, To>) { - // Polymorphic downcast. - return cel::internal::down_cast>( - std::forward(from)); - } else if constexpr (std::is_convertible_v && - !std::is_polymorphic_v && - !std::is_polymorphic_v>) { - return static_cast(std::forward(from)); - } else { - // Something else. - return std::forward(from).template Get(); - } - } - - template - ABSL_DEPRECATED( - "Use explicit conversion functions instead through static_cast.") - ABSL_MUST_USE_RESULT decltype(auto) - operator()(From* from) const { - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v, "From must be a non-union class"); - using R = decltype((*this)(*from)); - static_assert(std::is_lvalue_reference_v); - if (from == nullptr) { - return static_cast>>( - nullptr); - } - return static_cast>>( - std::addressof((*this)(*from))); - } -}; - -// Implementation of `cel::As`. -template -struct ABSL_DEPRECATED("Use As member functions instead.") AsImpl final { - static_assert(!std::is_pointer_v, "To must not be a pointer"); - static_assert(!std::is_array_v, "To must not be an array"); - static_assert(!std::is_lvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_rvalue_reference_v, - "To must not be a lvalue reference"); - static_assert(!std::is_const_v, "To must not be const qualified"); - static_assert(!std::is_volatile_v, "To must not be volatile qualified"); - static_assert(std::is_class_v, "To must be a non-union class"); - - explicit AsImpl() = default; - - template - ABSL_DEPRECATED("Use As member functions instead.") - ABSL_MUST_USE_RESULT decltype(auto) operator()(From&& from) const { - // Returns either `absl::optional` or `cel::optional_ref` - // depending on the return type of `CastTraits::Convert`. The use of these - // two types is an implementation detail. - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v>, - "From must be a non-union class"); - return std::forward(from).template As(); - } - - // Returns a pointer. - template - ABSL_DEPRECATED("Use As member functions instead.") - ABSL_MUST_USE_RESULT decltype(auto) operator()(From* from) const { - // Returns either `absl::optional` or `To*` depending on the return type of - // `CastTraits::Convert`. The use of these two types is an implementation - // detail. - static_assert(!std::is_volatile_v, - "From must not be volatile qualified"); - static_assert(std::is_class_v, "From must be a non-union class"); - using R = decltype(from->template As()); - if (from == nullptr) { - return R{absl::nullopt}; - } - return from->template As(); - } -}; - -} // namespace common_internal - -} // namespace cel - -#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_CASTING_H_ diff --git a/common/legacy_value.cc b/common/legacy_value.cc index d107c6f8a..505f2fbf5 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -36,7 +36,6 @@ #include "absl/types/span.h" #include "absl/types/variant.h" #include "base/attribute.h" -#include "common/casting.h" #include "common/kind.h" #include "common/memory.h" #include "common/type.h" @@ -1103,15 +1102,15 @@ absl::StatusOr LegacyValue( case ValueKind::kNull: return CelValue::CreateNull(); case ValueKind::kBool: - return CelValue::CreateBool(Cast(modern_value).NativeValue()); + return CelValue::CreateBool(modern_value.Get().NativeValue()); case ValueKind::kInt: - return CelValue::CreateInt64(Cast(modern_value).NativeValue()); + return CelValue::CreateInt64(modern_value.Get().NativeValue()); case ValueKind::kUint: return CelValue::CreateUint64( - Cast(modern_value).NativeValue()); + modern_value.Get().NativeValue()); case ValueKind::kDouble: return CelValue::CreateDouble( - Cast(modern_value).NativeValue()); + modern_value.Get().NativeValue()); case ValueKind::kString: return CelValue::CreateStringView(common_internal::LegacyStringValue( modern_value.GetString(), /*stable=*/false, arena)); @@ -1133,14 +1132,14 @@ absl::StatusOr LegacyValue( case ValueKind::kUnknown: return CelValue::CreateUnknownSet(google::protobuf::Arena::Create( arena, - common_internal::GetUnknown(Cast(modern_value)))); + common_internal::GetUnknown(modern_value.Get()))); case ValueKind::kType: return CelValue::CreateCelType( CelValue::CelTypeHolder(google::protobuf::Arena::Create( - arena, Cast(modern_value).NativeValue().name()))); + arena, modern_value.Get().NativeValue().name()))); case ValueKind::kError: return CelValue::CreateError(google::protobuf::Arena::Create( - arena, Cast(modern_value).NativeValue())); + arena, modern_value.Get().NativeValue())); default: return absl::InvalidArgumentError( absl::StrCat("google::api::expr::runtime::CelValue does not support ", @@ -1208,13 +1207,13 @@ absl::StatusOr ToLegacyValue( case ValueKind::kNull: return CelValue::CreateNull(); case ValueKind::kBool: - return CelValue::CreateBool(Cast(value).NativeValue()); + return CelValue::CreateBool(value.Get().NativeValue()); case ValueKind::kInt: - return CelValue::CreateInt64(Cast(value).NativeValue()); + return CelValue::CreateInt64(value.Get().NativeValue()); case ValueKind::kUint: - return CelValue::CreateUint64(Cast(value).NativeValue()); + return CelValue::CreateUint64(value.Get().NativeValue()); case ValueKind::kDouble: - return CelValue::CreateDouble(Cast(value).NativeValue()); + return CelValue::CreateDouble(value.Get().NativeValue()); case ValueKind::kString: return CelValue::CreateStringView(common_internal::LegacyStringValue( value.GetString(), /*stable=*/false, arena)); @@ -1225,24 +1224,24 @@ absl::StatusOr ToLegacyValue( return common_internal::LegacyTrivialStructValue(arena, value); case ValueKind::kDuration: return CelValue::CreateUncheckedDuration( - Cast(value).NativeValue()); + value.Get().NativeValue()); case ValueKind::kTimestamp: return CelValue::CreateTimestamp( - Cast(value).NativeValue()); + value.Get().NativeValue()); case ValueKind::kList: return common_internal::LegacyTrivialListValue(arena, value); case ValueKind::kMap: return common_internal::LegacyTrivialMapValue(arena, value); case ValueKind::kUnknown: return CelValue::CreateUnknownSet(google::protobuf::Arena::Create( - arena, common_internal::GetUnknown(Cast(value)))); + arena, common_internal::GetUnknown(value.Get()))); case ValueKind::kType: return CelValue::CreateCelType( CelValue::CelTypeHolder(google::protobuf::Arena::Create( - arena, Cast(value).NativeValue().name()))); + arena, value.Get().NativeValue().name()))); case ValueKind::kError: return CelValue::CreateError(google::protobuf::Arena::Create( - arena, Cast(value).NativeValue())); + arena, value.Get().NativeValue())); default: return absl::InvalidArgumentError( absl::StrCat("google::api::expr::runtime::CelValue does not support ", diff --git a/common/type_reflector_test.cc b/common/type_reflector_test.cc index 961de0d8e..52cd176ab 100644 --- a/common/type_reflector_test.cc +++ b/common/type_reflector_test.cc @@ -20,7 +20,6 @@ #include "absl/status/status_matchers.h" #include "absl/time/time.h" #include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "common/values/list_value.h" @@ -227,8 +226,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BoolValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), true); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), true); } TEST_F(TypeReflectorTest, NewValueBuilder_Int32Value) { @@ -261,8 +260,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int32Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kOutOfRange))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_Int64Value) { @@ -287,8 +286,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Int64Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_UInt32Value) { @@ -321,8 +320,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt32Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kOutOfRange))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_UInt64Value) { @@ -347,8 +346,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_UInt64Value) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_FloatValue) { @@ -373,8 +372,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_FloatValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_DoubleValue) { @@ -399,8 +398,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_DoubleValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), 1); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), 1); } TEST_F(TypeReflectorTest, NewValueBuilder_StringValue) { @@ -426,8 +425,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_StringValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeString(), "foo"); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeString(), "foo"); } TEST_F(TypeReflectorTest, NewValueBuilder_BytesValue) { @@ -453,8 +452,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_BytesValue) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeString(), "foo"); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeString(), "foo"); } TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { @@ -497,8 +496,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Duration) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), absl::Seconds(1) + absl::Nanoseconds(1)); } @@ -542,8 +541,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Timestamp) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), absl::UnixEpoch() + absl::Seconds(1) + absl::Nanoseconds(1)); } @@ -584,8 +583,8 @@ TEST_F(TypeReflectorTest, NewValueBuilder_Any) { IsOkAndHolds(Optional( ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument))))); ASSERT_OK_AND_ASSIGN(auto value, std::move(*builder).Build()); - EXPECT_TRUE(InstanceOf(value)); - EXPECT_EQ(Cast(value).NativeValue(), false); + EXPECT_TRUE(value.Is()); + EXPECT_EQ(value.Get().NativeValue(), false); } } // namespace diff --git a/common/values/custom_list_value.cc b/common/values/custom_list_value.cc index 303f49080..0e55115b0 100644 --- a/common/values/custom_list_value.cc +++ b/common/values/custom_list_value.cc @@ -24,7 +24,6 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "common/casting.h" #include "common/native_type.h" #include "common/value.h" #include "common/values/list_value_builder.h" @@ -334,7 +333,7 @@ absl::Status CustomListValueInterface::Contains( [&](size_t index, const Value& element) -> absl::StatusOr { CEL_RETURN_IF_ERROR(element.Equal(other, descriptor_pool, message_factory, arena, &equal)); - if (auto bool_result = As(equal); + if (auto bool_result = equal.AsBool(); bool_result.has_value() && bool_result->NativeValue()) { outcome = BoolValue(true); return false; @@ -596,7 +595,7 @@ absl::Status CustomListValue::Contains( [&](size_t index, const Value& element) -> absl::StatusOr { CEL_RETURN_IF_ERROR(element.Equal(other, descriptor_pool, message_factory, arena, &equal)); - if (auto bool_result = As(equal); + if (auto bool_result = equal.As(); bool_result.has_value() && bool_result->NativeValue()) { outcome = BoolValue(true); return false; diff --git a/common/values/list_value_test.cc b/common/values/list_value_test.cc index 321c05249..11a2b865d 100644 --- a/common/values/list_value_test.cc +++ b/common/values/list_value_test.cc @@ -21,7 +21,6 @@ #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "internal/testing.h" @@ -91,16 +90,16 @@ TEST_F(ListValueTest, Get) { NewIntListValue(IntValue(0), IntValue(1), IntValue(2))); ASSERT_OK_AND_ASSIGN(auto element, value.Get(0, descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - ASSERT_EQ(Cast(element).NativeValue(), 0); + ASSERT_TRUE(element.Is()); + ASSERT_EQ(element.Get().NativeValue(), 0); ASSERT_OK_AND_ASSIGN( element, value.Get(1, descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - ASSERT_EQ(Cast(element).NativeValue(), 1); + ASSERT_TRUE(element.Is()); + ASSERT_EQ(element.Get().NativeValue(), 1); ASSERT_OK_AND_ASSIGN( element, value.Get(2, descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - ASSERT_EQ(Cast(element).NativeValue(), 2); + ASSERT_TRUE(element.Is()); + ASSERT_EQ(element.Get().NativeValue(), 2); EXPECT_THAT( value.Get(3, descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kInvalidArgument)))); @@ -112,7 +111,7 @@ TEST_F(ListValueTest, ForEach) { std::vector elements; EXPECT_THAT(value.ForEach( [&elements](const Value& element) { - elements.push_back(Cast(element).NativeValue()); + elements.push_back(element.Get().NativeValue()); return true; }, descriptor_pool(), message_factory(), arena()), @@ -126,12 +125,12 @@ TEST_F(ListValueTest, Contains) { ASSERT_OK_AND_ASSIGN(auto contained, value.Contains(IntValue(2), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(contained)); - EXPECT_TRUE(Cast(contained).NativeValue()); + ASSERT_TRUE(contained.Is()); + EXPECT_TRUE(contained.Get().NativeValue()); ASSERT_OK_AND_ASSIGN(contained, value.Contains(IntValue(3), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(contained)); - EXPECT_FALSE(Cast(contained).NativeValue()); + ASSERT_TRUE(contained.Is()); + EXPECT_FALSE(contained.Get().NativeValue()); } TEST_F(ListValueTest, NewIterator) { @@ -143,8 +142,8 @@ TEST_F(ListValueTest, NewIterator) { ASSERT_OK_AND_ASSIGN( auto element, iterator->Next(descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - elements.push_back(Cast(element).NativeValue()); + ASSERT_TRUE(element.Is()); + elements.push_back(element.Get().NativeValue()); } EXPECT_EQ(iterator->HasNext(), false); EXPECT_THAT(iterator->Next(descriptor_pool(), message_factory(), arena()), diff --git a/common/values/map_value_test.cc b/common/values/map_value_test.cc index a1a82fd03..5b3deca27 100644 --- a/common/values/map_value_test.cc +++ b/common/values/map_value_test.cc @@ -22,7 +22,6 @@ #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "internal/testing.h" @@ -139,16 +138,16 @@ TEST_F(MapValueTest, Get) { std::pair{IntValue(2), DoubleValue(5.0)})); ASSERT_OK_AND_ASSIGN(auto value, map_value.Get(IntValue(0), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_EQ(Cast(value).NativeValue(), 3.0); + ASSERT_TRUE(value.Is()); + ASSERT_EQ(value.Get().NativeValue(), 3.0); ASSERT_OK_AND_ASSIGN(value, map_value.Get(IntValue(1), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_EQ(Cast(value).NativeValue(), 4.0); + ASSERT_TRUE(value.Is()); + ASSERT_EQ(value.Get().NativeValue(), 4.0); ASSERT_OK_AND_ASSIGN(value, map_value.Get(IntValue(2), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_EQ(Cast(value).NativeValue(), 5.0); + ASSERT_TRUE(value.Is()); + ASSERT_EQ(value.Get().NativeValue(), 5.0); EXPECT_THAT( map_value.Get(IntValue(3), descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound)))); @@ -164,18 +163,18 @@ TEST_F(MapValueTest, Find) { ASSERT_OK_AND_ASSIGN(entry, map_value.Find(IntValue(0), descriptor_pool(), message_factory(), arena())); ASSERT_TRUE(entry); - ASSERT_TRUE(InstanceOf(*entry)); - ASSERT_EQ(Cast(*entry).NativeValue(), 3.0); + ASSERT_TRUE(entry->Is()); + ASSERT_EQ(entry->Get().NativeValue(), 3.0); ASSERT_OK_AND_ASSIGN(entry, map_value.Find(IntValue(1), descriptor_pool(), message_factory(), arena())); ASSERT_TRUE(entry); - ASSERT_TRUE(InstanceOf(*entry)); - ASSERT_EQ(Cast(*entry).NativeValue(), 4.0); + ASSERT_TRUE(entry->Is()); + ASSERT_EQ(entry->Get().NativeValue(), 4.0); ASSERT_OK_AND_ASSIGN(entry, map_value.Find(IntValue(2), descriptor_pool(), message_factory(), arena())); ASSERT_TRUE(entry); - ASSERT_TRUE(InstanceOf(*entry)); - ASSERT_EQ(Cast(*entry).NativeValue(), 5.0); + ASSERT_TRUE(entry->Is()); + ASSERT_EQ(entry->Get().NativeValue(), 5.0); ASSERT_OK_AND_ASSIGN(entry, map_value.Find(IntValue(3), descriptor_pool(), message_factory(), arena())); ASSERT_FALSE(entry); @@ -189,20 +188,20 @@ TEST_F(MapValueTest, Has) { std::pair{IntValue(2), DoubleValue(5.0)})); ASSERT_OK_AND_ASSIGN(auto value, map_value.Has(IntValue(0), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_TRUE(Cast(value).NativeValue()); + ASSERT_TRUE(value.Is()); + ASSERT_TRUE(value.Get().NativeValue()); ASSERT_OK_AND_ASSIGN(value, map_value.Has(IntValue(1), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_TRUE(Cast(value).NativeValue()); + ASSERT_TRUE(value.Is()); + ASSERT_TRUE(value.Get().NativeValue()); ASSERT_OK_AND_ASSIGN(value, map_value.Has(IntValue(2), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_TRUE(Cast(value).NativeValue()); + ASSERT_TRUE(value.Is()); + ASSERT_TRUE(value.Get().NativeValue()); ASSERT_OK_AND_ASSIGN(value, map_value.Has(IntValue(3), descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(value)); - ASSERT_FALSE(Cast(value).NativeValue()); + ASSERT_TRUE(value.Is()); + ASSERT_FALSE(value.Get().NativeValue()); } TEST_F(MapValueTest, ListKeys) { @@ -217,7 +216,7 @@ TEST_F(MapValueTest, ListKeys) { std::vector keys; ASSERT_THAT(list_keys.ForEach( [&keys](const Value& element) -> bool { - keys.push_back(Cast(element).NativeValue()); + keys.push_back(element.Get().NativeValue()); return true; }, descriptor_pool(), message_factory(), arena()), @@ -235,8 +234,8 @@ TEST_F(MapValueTest, ForEach) { EXPECT_THAT(value.ForEach( [&entries](const Value& key, const Value& value) { entries.push_back( - std::pair{Cast(key).NativeValue(), - Cast(value).NativeValue()}); + std::pair{key.Get().NativeValue(), + value.Get().NativeValue()}); return true; }, descriptor_pool(), message_factory(), arena()), @@ -258,8 +257,8 @@ TEST_F(MapValueTest, NewIterator) { ASSERT_OK_AND_ASSIGN( auto element, iterator->Next(descriptor_pool(), message_factory(), arena())); - ASSERT_TRUE(InstanceOf(element)); - keys.push_back(Cast(element).NativeValue()); + ASSERT_TRUE(element.Is()); + keys.push_back(element.Get().NativeValue()); } EXPECT_EQ(iterator->HasNext(), false); EXPECT_THAT(iterator->Next(descriptor_pool(), message_factory(), arena()), diff --git a/common/values/null_value_test.cc b/common/values/null_value_test.cc index f217ddac4..ac40fef20 100644 --- a/common/values/null_value_test.cc +++ b/common/values/null_value_test.cc @@ -16,8 +16,6 @@ #include "absl/status/status_matchers.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "common/casting.h" #include "common/native_type.h" #include "common/value.h" #include "common/value_testing.h" @@ -27,8 +25,6 @@ namespace cel { namespace { using ::absl_testing::IsOk; -using ::testing::An; -using ::testing::Ne; using NullValueTest = common_internal::ValueTest<>; @@ -64,19 +60,5 @@ TEST_F(NullValueTest, NativeTypeId) { NativeTypeId::For()); } -TEST_F(NullValueTest, InstanceOf) { - EXPECT_TRUE(InstanceOf(NullValue())); - EXPECT_TRUE(InstanceOf(Value(NullValue()))); -} - -TEST_F(NullValueTest, Cast) { - EXPECT_THAT(Cast(NullValue()), An()); - EXPECT_THAT(Cast(Value(NullValue())), An()); -} - -TEST_F(NullValueTest, As) { - EXPECT_THAT(As(Value(NullValue())), Ne(std::nullopt)); -} - } // namespace } // namespace cel diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 8df9912bf..e71c1cdf2 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -543,7 +543,6 @@ cc_library( ":flat_expr_builder_extensions", "//base:builtins", "//common:ast", - "//common:casting", "//common:expr", "//common:native_type", "//common:value", @@ -551,7 +550,6 @@ cc_library( "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/eval:regex_match_step", - "//internal:casts", "//internal:re2_options", "//internal:status_macros", "@com_google_absl//absl/base:nullability", @@ -559,7 +557,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", "@com_googlesource_code_re2//:re2", ], ) diff --git a/eval/compiler/regex_precompilation_optimization.cc b/eval/compiler/regex_precompilation_optimization.cc index fa0d289f2..bd3a841ed 100644 --- a/eval/compiler/regex_precompilation_optimization.cc +++ b/eval/compiler/regex_precompilation_optimization.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -26,10 +27,8 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "base/builtins.h" #include "common/ast.h" -#include "common/casting.h" #include "common/expr.h" #include "common/native_type.h" #include "common/value.h" @@ -38,7 +37,6 @@ #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/regex_match_step.h" -#include "internal/casts.h" #include "internal/re2_options.h" #include "internal/status_macros.h" #include "re2/re2.h" @@ -48,14 +46,10 @@ namespace { using ::cel::Ast; using ::cel::CallExpr; -using ::cel::Cast; using ::cel::Expr; -using ::cel::InstanceOf; using ::cel::NativeTypeId; using ::cel::Reference; -using ::cel::StringValue; using ::cel::Value; -using ::cel::internal::down_cast; using ReferenceMap = absl::flat_hash_map; @@ -202,8 +196,8 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { } } - if (constant.has_value() && InstanceOf(*constant)) { - return Cast(*constant).ToString(); + if (constant.has_value() && constant->IsString()) { + return constant->GetString().ToString(); } return std::nullopt; diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 66d5092ab..6338b9461 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -63,7 +63,6 @@ cc_library( "//base:attributes", "//base:builtins", "//base:data", - "//common:casting", "//common:expr", "//common:function_descriptor", "//common:kind", @@ -252,7 +251,6 @@ cc_library( ":evaluator_core", ":expression_step_base", "//base:attributes", - "//common:casting", "//common:expr", "//common:kind", "//common:value", @@ -349,7 +347,6 @@ cc_library( ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:casting", "//common:expr", "//common:value", "//internal:status_macros", @@ -373,7 +370,6 @@ cc_library( ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:casting", "//common:value", "//internal:status_macros", "@com_google_absl//absl/container:flat_hash_set", @@ -397,7 +393,6 @@ cc_library( ":direct_expression_step", ":evaluator_core", ":expression_step_base", - "//common:casting", "//common:value", "//internal:status_macros", "@com_google_absl//absl/container:flat_hash_set", @@ -584,9 +579,7 @@ cc_test( ":attribute_trail", ":cel_expression_flat_impl", ":evaluator_core", - ":ident_step", "//base:data", - "//common:casting", "//common:memory", "//common:value", "//eval/public:activation", @@ -662,10 +655,8 @@ cc_test( ":const_value_step", ":direct_expression_step", ":evaluator_core", - ":ident_step", "//base:attributes", "//base:data", - "//common:casting", "//common:expr", "//common:unknown", "//common:value", @@ -703,11 +694,9 @@ cc_test( ":cel_expression_flat_impl", ":const_value_step", ":evaluator_core", - ":ident_step", ":select_step", "//base:attributes", "//base:data", - "//common:casting", "//common:expr", "//common:legacy_value", "//common:type", @@ -761,10 +750,8 @@ cc_test( ":create_list_step", ":direct_expression_step", ":evaluator_core", - ":ident_step", "//base:attributes", "//base:data", - "//common:casting", "//common:expr", "//common:value", "//common:value_testing", @@ -900,7 +887,6 @@ cc_library( "//base:function_result", "//base:function_result_set", "//base/internal:unknown_set", - "//common:casting", "//common:function_descriptor", "//common:unknown", "//common:value", @@ -972,18 +958,15 @@ cc_test( ":const_value_step", ":direct_expression_step", ":evaluator_core", - ":ident_step", ":ternary_step", "//base:attributes", "//base:data", - "//common:casting", "//common:expr", "//common:value", "//eval/public:activation", "//eval/public:cel_value", "//eval/public:unknown_attribute_set", "//eval/public:unknown_set", - "//internal:status_macros", "//internal:testing", "//internal:testing_descriptor_pool", "//internal:testing_message_factory", @@ -1130,8 +1113,6 @@ cc_library( ":evaluator_core", ":expression_step_base", ":expression_step_logic", - "//common:casting", - "//common:optional_ref", "//common:value", "//internal:status_macros", "//runtime/internal:errors", @@ -1151,7 +1132,6 @@ cc_test( ":direct_expression_step", ":evaluator_core", ":optional_or_step", - "//common:casting", "//common:value", "//common:value_kind", "//common:value_testing", diff --git a/eval/eval/attribute_utility.cc b/eval/eval/attribute_utility.cc index af63e9f91..a0e57161a 100644 --- a/eval/eval/attribute_utility.cc +++ b/eval/eval/attribute_utility.cc @@ -1,6 +1,7 @@ #include "eval/eval/attribute_utility.h" #include +#include #include #include @@ -12,7 +13,6 @@ #include "base/function_result.h" #include "base/function_result_set.h" #include "base/internal/unknown_set.h" -#include "common/casting.h" #include "common/function_descriptor.h" #include "common/unknown.h" #include "common/value.h" @@ -27,11 +27,9 @@ namespace google::api::expr::runtime { using ::cel::Attribute; using ::cel::AttributePattern; using ::cel::AttributeSet; -using ::cel::Cast; using ::cel::ErrorValue; using ::cel::FunctionResult; using ::cel::FunctionResultSet; -using ::cel::InstanceOf; using ::cel::UnknownValue; using ::cel::Value; using ::cel::base_internal::UnknownSet; diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc index 6293f97f4..661a8f2e4 100644 --- a/eval/eval/comprehension_step.cc +++ b/eval/eval/comprehension_step.cc @@ -13,7 +13,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "base/attribute.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" @@ -33,8 +32,6 @@ enum class IterableKind { }; using ::cel::AttributeQualifier; -using ::cel::Cast; -using ::cel::InstanceOf; using ::cel::UnknownValue; using ::cel::Value; using ::cel::ValueIterator; diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index f4a33b217..dc1efbd12 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -11,7 +11,6 @@ #include "absl/types/optional.h" #include "absl/types/span.h" #include "base/attribute.h" -#include "common/casting.h" #include "common/expr.h" #include "common/kind.h" #include "common/value.h" @@ -31,9 +30,7 @@ namespace google::api::expr::runtime { namespace { using ::cel::AttributeQualifier; -using ::cel::Cast; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::IntValue; using ::cel::ListValue; using ::cel::MapValue; @@ -170,7 +167,7 @@ void LookupInList(const ListValue& cel_list, const Value& key, if (number.has_value() && number->LosslessConvertibleToInt()) { maybe_idx = number->AsInt(); } - } else if (InstanceOf(key)) { + } else if (key.IsInt()) { maybe_idx = key.GetInt().NativeValue(); } @@ -212,11 +209,11 @@ void LookupInContainer(const Value& container, const Value& key, // Select steps can be applied to either maps or messages switch (container.kind()) { case ValueKind::kMap: { - LookupInMap(Cast(container), key, frame, result); + LookupInMap(container.GetMap(), key, frame, result); return; } case ValueKind::kList: { - LookupInList(Cast(container), key, frame, result); + LookupInList(container.GetList(), key, frame, result); return; } default: @@ -252,11 +249,11 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, } } - if (InstanceOf(container)) { + if (container.IsError()) { result = container; return; } - if (InstanceOf(key)) { + if (key.IsError()) { result = key; return; } @@ -270,7 +267,7 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, Value value; optional_value.Value(&value); LookupInContainer(value, key, frame, result); - if (auto error_value = cel::As(result); + if (auto error_value = result.AsError(); error_value && cel::IsNoSuchKey(*error_value)) { result = cel::OptionalValue::None(); return; diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index 9e8344fcf..8a7470fa7 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -10,7 +10,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/types/optional.h" -#include "common/casting.h" #include "common/expr.h" #include "common/value.h" #include "common/values/list_value_builder.h" @@ -25,9 +24,7 @@ namespace google::api::expr::runtime { namespace { -using ::cel::Cast; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::ListValueBuilderPtr; using ::cel::UnknownValue; using ::cel::Value; diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index 4475c1fb2..119504a14 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -14,7 +14,6 @@ #include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/expr.h" #include "common/value.h" #include "common/value_testing.h" @@ -51,12 +50,9 @@ using ::absl_testing::StatusIs; using ::cel::Attribute; using ::cel::AttributeQualifier; using ::cel::AttributeSet; -using ::cel::Cast; using ::cel::ErrorValue; using ::cel::Expr; -using ::cel::InstanceOf; using ::cel::IntValue; -using ::cel::ListValue; using ::cel::TypeProvider; using ::cel::UnknownValue; using ::cel::Value; @@ -289,8 +285,8 @@ TEST(CreateDirectListStep, Basic) { ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).Size(), IsOkAndHolds(2)); + ASSERT_TRUE(result.IsList()); + EXPECT_THAT(result.GetList().Size(), IsOkAndHolds(2)); } TEST(CreateDirectListStep, ForwardFirstError) { @@ -317,8 +313,8 @@ TEST(CreateDirectListStep, ForwardFirstError) { ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), + ASSERT_TRUE(result.IsError()); + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kInternal, "test1")); } @@ -366,8 +362,8 @@ TEST(CreateDirectListStep, MergeUnknowns) { ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(UnknownAttrNames(Cast(result)), + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(UnknownAttrNames(result.GetUnknown()), UnorderedElementsAre("var1", "var2")); } @@ -397,8 +393,8 @@ TEST(CreateDirectListStep, ErrorBeforeUnknown) { ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), + ASSERT_TRUE(result.IsError()); + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kInternal, "test1")); } @@ -445,9 +441,9 @@ TEST(CreateDirectListStep, MissingAttribute) { ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); + ASSERT_TRUE(result.IsError()); EXPECT_THAT( - Cast(result).NativeValue(), + result.GetError().NativeValue(), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("var1.field1"))); } @@ -474,8 +470,8 @@ TEST(CreateDirectListStep, OptionalPresentSet) { ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - auto list = Cast(result); + ASSERT_TRUE(result.IsList()); + auto list = result.GetList(); EXPECT_THAT(list.Size(), IsOkAndHolds(2)); EXPECT_THAT(list.Get(0, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena), @@ -507,8 +503,8 @@ TEST(CreateDirectListStep, OptionalAbsentNotSet) { ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - auto list = Cast(result); + ASSERT_TRUE(result.IsList()); + auto list = result.GetList(); EXPECT_THAT(list.Size(), IsOkAndHolds(1)); EXPECT_THAT(list.Get(0, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena), @@ -540,8 +536,8 @@ TEST(CreateDirectListStep, PartialUnknown) { ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(UnknownAttrNames(Cast(result)), + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(UnknownAttrNames(result.GetUnknown()), UnorderedElementsAre("var1")); } diff --git a/eval/eval/create_map_step.cc b/eval/eval/create_map_step.cc index f8f099047..15e696782 100644 --- a/eval/eval/create_map_step.cc +++ b/eval/eval/create_map_step.cc @@ -25,7 +25,6 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "common/values/map_value_builder.h" #include "eval/eval/attribute_trail.h" @@ -38,11 +37,9 @@ namespace google::api::expr::runtime { namespace { -using ::cel::Cast; using ::cel::ErrorValue; using ::cel::ErrorValueAssign; using ::cel::ErrorValueReturn; -using ::cel::InstanceOf; using ::cel::MapValueBuilderPtr; using ::cel::UnknownValue; using ::cel::Value; diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index 3ac6caabe..8fcdd3c5a 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -26,7 +26,6 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/direct_expression_step.h" @@ -38,9 +37,7 @@ namespace google::api::expr::runtime { namespace { -using ::cel::Cast; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::StructValueBuilderInterface; using ::cel::UnknownValue; using ::cel::Value; diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index 91a9fa87f..f3d3fb93a 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -16,7 +16,6 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/span.h" -#include "common/casting.h" #include "common/expr.h" #include "common/function_descriptor.h" #include "common/kind.h" @@ -179,7 +178,7 @@ Value NoOverloadResult(absl::string_view name, // should be propagated along execution path. for (size_t i = 0; i < args.size(); i++) { const auto& arg = args[i]; - if (cel::InstanceOf(arg)) { + if (arg.IsError()) { return arg; } } diff --git a/eval/eval/ident_step_test.cc b/eval/eval/ident_step_test.cc index 91b73a0fd..f27eaf477 100644 --- a/eval/eval/ident_step_test.cc +++ b/eval/eval/ident_step_test.cc @@ -7,7 +7,6 @@ #include "absl/status/status.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/memory.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" @@ -31,9 +30,7 @@ namespace { using ::absl_testing::IsOk; using ::absl_testing::StatusIs; -using ::cel::Cast; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::IntValue; using ::cel::MemoryManagerRef; using ::cel::RuntimeOptions; @@ -224,8 +221,8 @@ TEST(DirectIdentStepTest, Basic) { ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), Eq(42)); + ASSERT_TRUE(result.IsInt()); + EXPECT_THAT(result.GetInt().NativeValue(), Eq(42)); } TEST(DirectIdentStepTest, UnknownAttribute) { @@ -249,8 +246,8 @@ TEST(DirectIdentStepTest, UnknownAttribute) { ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).ToAttributeSet(), SizeIs(1)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), SizeIs(1)); } TEST(DirectIdentStepTest, MissingAttribute) { @@ -274,8 +271,8 @@ TEST(DirectIdentStepTest, MissingAttribute) { ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), + ASSERT_TRUE(result.IsError()); + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("var1"))); } @@ -296,8 +293,8 @@ TEST(DirectIdentStepTest, NotFound) { ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), + ASSERT_TRUE(result.IsError()); + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kUnknown, HasSubstr("\"var1\" found in Activation"))); } diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index bd6803059..d7bab5f8c 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -10,7 +10,6 @@ #include "absl/strings/string_view.h" #include "absl/types/span.h" #include "base/builtins.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" @@ -25,9 +24,7 @@ namespace google::api::expr::runtime { namespace { using ::cel::BoolValue; -using ::cel::Cast; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::UnknownValue; using ::cel::Value; using ::cel::ValueKind; @@ -45,7 +42,7 @@ absl::Status ReturnLogicResult(ExecutionFrameBase& frame, BoolLogicKind op_type, if (frame.unknown_processing_enabled()) { if (lhs_kind == ValueKind::kUnknown && rhs_kind == ValueKind::kUnknown) { lhs_result = frame.attribute_utility().MergeUnknownValues( - Cast(lhs_result), Cast(rhs_result)); + lhs_result.GetUnknown(), rhs_result.GetUnknown()); // Clear attribute trail so this doesn't get re-identified as a new // unknown and reset the accumulated attributes. attribute_trail = AttributeTrail(); @@ -112,7 +109,7 @@ absl::Status ExhaustiveDirectLogicStep::Evaluate( ValueKind rhs_kind = rhs_result.kind(); if (lhs_kind == ValueKind::kBool) { - bool lhs_bool = Cast(result).NativeValue(); + bool lhs_bool = result.GetBool().NativeValue(); if ((op_type_ == BoolLogicKind::kOr && lhs_bool) || (op_type_ == BoolLogicKind::kAnd && !lhs_bool)) { return absl::OkStatus(); @@ -120,7 +117,7 @@ absl::Status ExhaustiveDirectLogicStep::Evaluate( } if (rhs_kind == ValueKind::kBool) { - bool rhs_bool = Cast(rhs_result).NativeValue(); + bool rhs_bool = rhs_result.GetBool().NativeValue(); if ((op_type_ == BoolLogicKind::kOr && rhs_bool) || (op_type_ == BoolLogicKind::kAnd && !rhs_bool)) { result = std::move(rhs_result); @@ -157,7 +154,7 @@ absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, CEL_RETURN_IF_ERROR(lhs_->Evaluate(frame, result, attribute_trail)); ValueKind lhs_kind = result.kind(); if (lhs_kind == ValueKind::kBool) { - bool lhs_bool = Cast(result).NativeValue(); + bool lhs_bool = result.GetBool().NativeValue(); if ((op_type_ == BoolLogicKind::kOr && lhs_bool) || (op_type_ == BoolLogicKind::kAnd && !lhs_bool)) { return absl::OkStatus(); @@ -172,7 +169,7 @@ absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, ValueKind rhs_kind = rhs_result.kind(); if (rhs_kind == ValueKind::kBool) { - bool rhs_bool = Cast(rhs_result).NativeValue(); + bool rhs_bool = rhs_result.GetBool().NativeValue(); if ((op_type_ == BoolLogicKind::kOr && rhs_bool) || (op_type_ == BoolLogicKind::kAnd && !rhs_bool)) { result = std::move(rhs_result); diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index b289e0e0a..04b9fd127 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -14,7 +14,6 @@ #include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/expr.h" #include "common/unknown.h" #include "common/value.h" @@ -48,9 +47,7 @@ using ::absl_testing::IsOk; using ::cel::Attribute; using ::cel::AttributeSet; using ::cel::BoolValue; -using ::cel::Cast; using ::cel::Expr; -using ::cel::InstanceOf; using ::cel::IntValue; using ::cel::TypeProvider; using ::cel::UnknownValue; diff --git a/eval/eval/optional_or_step.cc b/eval/eval/optional_or_step.cc index 5ac1e8400..983f229b1 100644 --- a/eval/eval/optional_or_step.cc +++ b/eval/eval/optional_or_step.cc @@ -22,8 +22,6 @@ #include "absl/base/optimization.h" #include "absl/status/status.h" #include "absl/types/span.h" -#include "common/casting.h" -#include "common/optional_ref.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/direct_expression_step.h" @@ -38,11 +36,7 @@ namespace google::api::expr::runtime { namespace { -using ::cel::As; using ::cel::ErrorValue; -using ::cel::InstanceOf; -using ::cel::OptionalValue; -using ::cel::UnknownValue; using ::cel::Value; using ::cel::runtime_internal::CreateNoMatchingOverloadError; @@ -78,13 +72,13 @@ absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, const Value& rhs, const AttributeTrail& lhs_attr, const AttributeTrail& rhs_attr, Value& result, AttributeTrail& result_attr, google::protobuf::Arena* arena) { - if (InstanceOf(lhs) || InstanceOf(lhs)) { + if (lhs.IsError() || lhs.IsUnknown()) { result = lhs; result_attr = lhs_attr; return absl::OkStatus(); } - auto lhs_optional_value = As(lhs); + auto lhs_optional_value = lhs.AsOptional(); if (!lhs_optional_value.has_value()) { result = MakeNoOverloadError(kind, arena); result_attr = AttributeTrail(); @@ -101,8 +95,8 @@ absl::Status EvalOptionalOr(OptionalOrKind kind, const Value& lhs, return absl::OkStatus(); } - if (kind == OptionalOrKind::kOrOptional && !InstanceOf(rhs) && - !InstanceOf(rhs) && !InstanceOf(rhs)) { + if (kind == OptionalOrKind::kOrOptional && !rhs.IsError() && + !rhs.IsUnknown() && !rhs.IsOptional()) { result = MakeNoOverloadError(kind, arena); result_attr = AttributeTrail(); return absl::OkStatus(); @@ -193,13 +187,13 @@ absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, AttributeTrail& attribute) const { CEL_RETURN_IF_ERROR(optional_->Evaluate(frame, result, attribute)); - if (InstanceOf(result) || InstanceOf(result)) { + if (result.IsUnknown() || result.IsError()) { // Forward the lhs error instead of attempting to evaluate the alternative // (unlike CEL's commutative logic operators). return absl::OkStatus(); } - auto optional_value = As(static_cast(result)); + auto optional_value = result.AsOptional(); if (!optional_value.has_value()) { result = MakeNoOverloadError(kind_, frame.arena()); return absl::OkStatus(); @@ -218,8 +212,7 @@ absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, // // Otherwise, we don't know what type to expect so can't check anything. if (kind_ == OptionalOrKind::kOrOptional) { - if (!InstanceOf(result) && !InstanceOf(result) && - !InstanceOf(result)) { + if (!result.IsOptional() && !result.IsError() && !result.IsUnknown()) { result = MakeNoOverloadError(kind_, frame.arena()); } } @@ -235,15 +228,13 @@ void OptionalHasValueJumpStep::Evaluate(ExecutionFrame* frame) const { return; } const Value& value = frame->value_stack().Peek(); - cel::optional_ref optional_value = - As(value); + auto optional_value = value.AsOptional(); // We jump if the receiver is `optional_type` which has a value or the // receiver is an error/unknown. Unlike `_||_` we are not commutative. If // we run into an error/unknown, we skip the `else` branch. const bool should_jump = (optional_value.has_value() && optional_value->HasValue()) || - (!optional_value.has_value() && (cel::InstanceOf(value) || - cel::InstanceOf(value))); + (!optional_value.has_value() && (value.IsError() || value.IsUnknown())); if (should_jump) { if (is_or_value_ && optional_value.has_value()) { frame->value_stack().PopAndPush(optional_value->Value()); diff --git a/eval/eval/optional_or_step_test.cc b/eval/eval/optional_or_step_test.cc index 2962ee38c..f0641ca6c 100644 --- a/eval/eval/optional_or_step_test.cc +++ b/eval/eval/optional_or_step_test.cc @@ -18,7 +18,6 @@ #include "absl/memory/memory.h" #include "absl/status/status.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_kind.h" #include "common/value_testing.h" @@ -40,9 +39,7 @@ namespace { using ::absl_testing::StatusIs; using ::cel::Activation; -using ::cel::As; using ::cel::ErrorValue; -using ::cel::InstanceOf; using ::cel::IntValue; using ::cel::OptionalValue; using ::cel::RuntimeOptions; diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index 3ab35fd63..a1e5b4a09 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -17,7 +17,6 @@ #include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/expr.h" #include "common/legacy_value.h" #include "common/type.h" @@ -62,10 +61,8 @@ using ::cel::Attribute; using ::cel::AttributeQualifier; using ::cel::AttributeSet; using ::cel::BoolValue; -using ::cel::Cast; using ::cel::ErrorValue; using ::cel::Expr; -using ::cel::InstanceOf; using ::cel::IntValue; using ::cel::OptionalValue; using ::cel::RuntimeOptions; @@ -1140,9 +1137,9 @@ TEST_F(DirectSelectStepTest, SelectFromMap) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); + ASSERT_TRUE(result.IsInt()); - EXPECT_EQ(Cast(result).NativeValue(), 1); + EXPECT_EQ(result.GetInt().NativeValue(), 1); } TEST_F(DirectSelectStepTest, HasMap) { @@ -1171,9 +1168,9 @@ TEST_F(DirectSelectStepTest, HasMap) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); + ASSERT_TRUE(result.GetBool()); - EXPECT_TRUE(Cast(result).NativeValue()); + EXPECT_TRUE(result.GetBool().NativeValue()); } TEST_F(DirectSelectStepTest, SelectFromOptionalMap) { @@ -1204,9 +1201,8 @@ TEST_F(DirectSelectStepTest, SelectFromOptionalMap) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(static_cast(result)).Value(), - IntValueIs(1)); + ASSERT_TRUE(result.IsOptional()); + EXPECT_THAT(result.GetOptional().Value(), IntValueIs(1)); } TEST_F(DirectSelectStepTest, SelectFromOptionalMapAbsent) { @@ -1237,9 +1233,8 @@ TEST_F(DirectSelectStepTest, SelectFromOptionalMapAbsent) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_FALSE( - Cast(static_cast(result)).HasValue()); + ASSERT_TRUE(result.IsOptional()); + EXPECT_FALSE(result.GetOptional().HasValue()); } TEST_F(DirectSelectStepTest, SelectFromOptionalStruct) { @@ -1272,9 +1267,8 @@ TEST_F(DirectSelectStepTest, SelectFromOptionalStruct) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(static_cast(result)).Value(), - IntValueIs(1)); + ASSERT_TRUE(result.IsOptional()); + EXPECT_THAT(result.GetOptional().Value(), IntValueIs(1)); } TEST_F(DirectSelectStepTest, SelectFromOptionalStructFieldNotSet) { @@ -1307,9 +1301,8 @@ TEST_F(DirectSelectStepTest, SelectFromOptionalStructFieldNotSet) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_FALSE( - Cast(static_cast(result)).HasValue()); + ASSERT_TRUE(result.IsOptional()); + EXPECT_FALSE(result.GetOptional().HasValue()); } TEST_F(DirectSelectStepTest, SelectFromEmptyOptional) { @@ -1332,9 +1325,8 @@ TEST_F(DirectSelectStepTest, SelectFromEmptyOptional) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_FALSE( - cel::Cast(static_cast(result)).HasValue()); + ASSERT_TRUE(result.IsOptional()); + EXPECT_FALSE(result.GetOptional().HasValue()); } TEST_F(DirectSelectStepTest, HasOptional) { @@ -1365,9 +1357,9 @@ TEST_F(DirectSelectStepTest, HasOptional) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); + ASSERT_TRUE(result.IsBool()); - EXPECT_TRUE(Cast(result).NativeValue()); + EXPECT_TRUE(result.GetBool().NativeValue()); } TEST_F(DirectSelectStepTest, HasEmptyOptional) { @@ -1390,9 +1382,9 @@ TEST_F(DirectSelectStepTest, HasEmptyOptional) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); + ASSERT_TRUE(result.IsBool()); - EXPECT_FALSE(Cast(result).NativeValue()); + EXPECT_FALSE(result.GetBool().NativeValue()); } TEST_F(DirectSelectStepTest, SelectFromStruct) { @@ -1416,9 +1408,9 @@ TEST_F(DirectSelectStepTest, SelectFromStruct) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); + ASSERT_TRUE(result.IsInt()); - EXPECT_EQ(Cast(result).NativeValue(), 1); + EXPECT_EQ(result.GetInt().NativeValue(), 1); } TEST_F(DirectSelectStepTest, HasStruct) { @@ -1444,8 +1436,8 @@ TEST_F(DirectSelectStepTest, HasStruct) { // has(test_all_types.single_string) ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_FALSE(Cast(result).NativeValue()); + ASSERT_TRUE(result.IsBool()); + EXPECT_FALSE(result.GetBool().NativeValue()); } TEST_F(DirectSelectStepTest, SelectFromUnsupportedType) { @@ -1467,9 +1459,9 @@ TEST_F(DirectSelectStepTest, SelectFromUnsupportedType) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); + ASSERT_TRUE(result.IsError()); - EXPECT_THAT(Cast(result).NativeValue(), + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Applying SELECT to non-message type"))); } @@ -1496,8 +1488,8 @@ TEST_F(DirectSelectStepTest, AttributeUpdatedIfRequested) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_EQ(Cast(result).NativeValue(), 1); + ASSERT_TRUE(result.IsInt()); + EXPECT_EQ(result.GetInt().NativeValue(), 1); ASSERT_OK_AND_ASSIGN(std::string attr_str, attr.attribute().AsString()); EXPECT_EQ(attr_str, "test_all_types.single_int64"); @@ -1528,8 +1520,8 @@ TEST_F(DirectSelectStepTest, MissingAttributesToErrors) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), + ASSERT_TRUE(result.IsError()); + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("test_all_types.single_int64"))); } @@ -1559,9 +1551,9 @@ TEST_F(DirectSelectStepTest, IdentifiesUnknowns) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); + ASSERT_TRUE(result.IsUnknown()); - EXPECT_THAT(AttributeStrings(Cast(result)), + EXPECT_THAT(AttributeStrings(result.GetUnknown()), UnorderedElementsAre("test_all_types.single_int64")); } @@ -1585,8 +1577,8 @@ TEST_F(DirectSelectStepTest, ForwardErrorValue) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), + ASSERT_TRUE(result.IsError()); + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kInternal, HasSubstr("test1"))); } @@ -1616,8 +1608,8 @@ TEST_F(DirectSelectStepTest, ForwardUnknownOperand) { AttributeTrail attr; ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(AttributeStrings(Cast(result)), + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(AttributeStrings(result.GetUnknown()), UnorderedElementsAre("attr[0]")); } diff --git a/eval/eval/ternary_step_test.cc b/eval/eval/ternary_step_test.cc index cedab0ca2..2d49700cc 100644 --- a/eval/eval/ternary_step_test.cc +++ b/eval/eval/ternary_step_test.cc @@ -10,7 +10,6 @@ #include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/casting.h" #include "common/expr.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" @@ -18,12 +17,10 @@ #include "eval/eval/const_value_step.h" #include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_value.h" #include "eval/public/unknown_attribute_set.h" #include "eval/public/unknown_set.h" -#include "internal/status_macros.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "internal/testing_message_factory.h" @@ -41,10 +38,8 @@ namespace { using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::BoolValue; -using ::cel::Cast; using ::cel::ErrorValue; using ::cel::Expr; -using ::cel::InstanceOf; using ::cel::IntValue; using ::cel::RuntimeOptions; using ::cel::TypeProvider; @@ -227,8 +222,8 @@ TEST_P(TernaryStepDirectTest, ReturnLhs) { ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_EQ(Cast(result).NativeValue(), 1); + ASSERT_TRUE(result.IsInt()); + EXPECT_EQ(result.GetInt().NativeValue(), 1); } TEST_P(TernaryStepDirectTest, ReturnRhs) { @@ -248,8 +243,8 @@ TEST_P(TernaryStepDirectTest, ReturnRhs) { ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_EQ(Cast(result).NativeValue(), 2); + ASSERT_TRUE(result.IsInt()); + EXPECT_EQ(result.GetInt().NativeValue(), 2); } TEST_P(TernaryStepDirectTest, ForwardError) { @@ -272,8 +267,8 @@ TEST_P(TernaryStepDirectTest, ForwardError) { ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), + ASSERT_TRUE(result.IsError()); + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kInternal, "test error")); } @@ -299,8 +294,8 @@ TEST_P(TernaryStepDirectTest, ForwardUnknown) { AttributeTrail attr_unused; ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).ToAttributeSet(), + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), ElementsAre(Truly([](const cel::Attribute& attr) { return attr.variable_name() == "var"; }))); @@ -323,8 +318,8 @@ TEST_P(TernaryStepDirectTest, UnexpectedCondtionKind) { ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), + ASSERT_TRUE(result.IsError()); + EXPECT_THAT(result.GetError().NativeValue(), StatusIs(absl::StatusCode::kUnknown, HasSubstr("No matching overloads found"))); } @@ -364,8 +359,8 @@ TEST_P(TernaryStepDirectTest, Shortcircuiting) { ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - ASSERT_TRUE(InstanceOf(result)); - EXPECT_THAT(Cast(result).NativeValue(), Eq(1)); + ASSERT_TRUE(result.IsInt()); + EXPECT_THAT(result.GetInt().NativeValue(), Eq(1)); bool expect_eager_eval = !Shortcircuiting(); EXPECT_EQ(lhs_was_called, expect_eager_eval); EXPECT_TRUE(rhs_was_called); diff --git a/eval/tests/BUILD b/eval/tests/BUILD index e4dacea3f..b706101df 100644 --- a/eval/tests/BUILD +++ b/eval/tests/BUILD @@ -69,8 +69,6 @@ cc_test( deps = [ ":request_context_cc_proto", "//checker:validation_result", - "//common:allocator", - "//common:casting", "//common:decl", "//common:legacy_value", "//common:native_type", diff --git a/eval/tests/modern_benchmark_test.cc b/eval/tests/modern_benchmark_test.cc index 1d7731e6a..de984617e 100644 --- a/eval/tests/modern_benchmark_test.cc +++ b/eval/tests/modern_benchmark_test.cc @@ -37,8 +37,6 @@ #include "absl/strings/match.h" #include "absl/strings/string_view.h" #include "checker/validation_result.h" -#include "common/allocator.h" -#include "common/casting.h" #include "common/decl.h" #include "common/native_type.h" #include "common/type.h" @@ -164,8 +162,8 @@ static void BM_Eval(benchmark::State& state) { Activation activation; ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_TRUE(Cast(result) == len + 1); + ASSERT_TRUE(result.Is()); + ASSERT_TRUE(result.Get() == len + 1); } } @@ -210,8 +208,8 @@ static void BM_Eval_Trace(benchmark::State& state) { Activation activation; ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_TRUE(Cast(result) == len + 1); + ASSERT_TRUE(result.Is()); + ASSERT_TRUE(result.Get() == len + 1); } } @@ -250,8 +248,8 @@ static void BM_EvalString(benchmark::State& state) { Activation activation; ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_TRUE(Cast(result).Size() == len + 1); + ASSERT_TRUE(result.Is()); + ASSERT_TRUE(result.Get().Size() == len + 1); } } @@ -291,8 +289,8 @@ static void BM_EvalString_Trace(benchmark::State& state) { Activation activation; ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_TRUE(Cast(result).Size() == len + 1); + ASSERT_TRUE(result.Is()); + ASSERT_TRUE(result.Get().Size() == len + 1); } } @@ -390,7 +388,7 @@ void BM_PolicySymbolic(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - auto result_bool = As(result); + auto result_bool = result.As(); ASSERT_TRUE(result_bool && result_bool->NativeValue()); } } @@ -434,7 +432,7 @@ class RequestMapImpl : public CustomMapValueInterface { google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena, Value* absl_nonnull result) const override { - auto string_value = As(key); + auto string_value = key.As(); if (!string_value) { return false; } @@ -503,8 +501,8 @@ void BM_PolicySymbolicMap(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -630,8 +628,8 @@ void BM_Comprehension(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_EQ(Cast(result), len); + ASSERT_TRUE(result.Is()); + ASSERT_EQ(result.Get(), len); } } @@ -663,8 +661,8 @@ void BM_Comprehension_Trace(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_EQ(Cast(result), len); + ASSERT_TRUE(result.Is()); + ASSERT_EQ(result.Get(), len); } } @@ -693,8 +691,8 @@ void BM_HasMap(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -733,8 +731,8 @@ void BM_HasProto(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -772,8 +770,8 @@ void BM_HasProtoMap(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -812,8 +810,8 @@ void BM_ReadProtoMap(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -852,8 +850,8 @@ void BM_NestedProtoFieldRead(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -891,8 +889,8 @@ void BM_NestedProtoFieldReadDefaults(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -934,8 +932,8 @@ void BM_ProtoStructAccess(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -980,8 +978,8 @@ void BM_ProtoListAccess(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result) && - Cast(result).NativeValue()); + ASSERT_TRUE(result.Is() && + result.Get().NativeValue()); } } @@ -1108,8 +1106,8 @@ void BM_NestedComprehension(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_EQ(Cast(result), len * len); + ASSERT_TRUE(result.Is()); + ASSERT_EQ(result.Get(), len * len); } } @@ -1145,8 +1143,8 @@ void BM_NestedComprehension_Trace(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, &EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_EQ(Cast(result), len * len); + ASSERT_TRUE(result.Is()); + ASSERT_EQ(result.Get(), len * len); } } @@ -1179,8 +1177,8 @@ void BM_ListComprehension(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_THAT(Cast(result).Size(), IsOkAndHolds(len)); + ASSERT_TRUE(result.Is()); + ASSERT_THAT(result.Get().Size(), IsOkAndHolds(len)); } } @@ -1214,8 +1212,8 @@ void BM_ListComprehension_Trace(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Trace(&arena, activation, EmptyCallback)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_THAT(Cast(result).Size(), IsOkAndHolds(len)); + ASSERT_TRUE(result.Is()); + ASSERT_THAT(result.Get().Size(), IsOkAndHolds(len)); } } @@ -1378,8 +1376,8 @@ void BM_ListComprehension_Opt(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_THAT(Cast(result).Size(), IsOkAndHolds(len)); + ASSERT_TRUE(result.Is()); + ASSERT_THAT(result.Get().Size(), IsOkAndHolds(len)); } } @@ -1399,7 +1397,7 @@ void BM_ComprehensionCpp(benchmark::State& state) { auto op = [&list]() { int sum = 0; for (const auto& value : list) { - sum += Cast(value).NativeValue(); + sum += value.Get().NativeValue(); } return sum; }; @@ -1460,8 +1458,8 @@ void BM_MapTransformComprehension(benchmark::State& state) { for (auto _ : state) { ASSERT_OK_AND_ASSIGN(cel::Value result, cel_expr->Evaluate(&arena, activation)); - ASSERT_TRUE(InstanceOf(result)); - ASSERT_THAT(Cast(result).Size(), IsOkAndHolds(len)); + ASSERT_TRUE(result.Is()); + ASSERT_THAT(result.Get().Size(), IsOkAndHolds(len)); } } diff --git a/extensions/BUILD b/extensions/BUILD index 743d32a9b..908cacb18 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -76,7 +76,6 @@ cc_library( hdrs = ["math_ext.h"], deps = [ ":math_ext_decls", - "//common:casting", "//common:value", "//eval/public:cel_function_registry", "//eval/public:cel_number", @@ -327,7 +326,6 @@ cc_library( "//base:builtins", "//common:ast", "//common:ast_rewrite", - "//common:casting", "//common:constant", "//common:expr", "//common:function_descriptor", diff --git a/extensions/math_ext.cc b/extensions/math_ext.cc index 78c066f9f..49a0d2b38 100644 --- a/extensions/math_ext.cc +++ b/extensions/math_ext.cc @@ -25,7 +25,6 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "common/casting.h" #include "common/value.h" #include "eval/public/cel_function_registry.h" #include "eval/public/cel_number.h" @@ -62,13 +61,13 @@ Value NumberToValue(CelNumber number) { absl::StatusOr ValueToNumber(const Value& value, absl::string_view function) { - if (auto int_value = As(value); int_value) { + if (auto int_value = value.AsInt(); int_value) { return CelNumber::FromInt64(int_value->NativeValue()); } - if (auto uint_value = As(value); uint_value) { + if (auto uint_value = value.AsUint(); uint_value) { return CelNumber::FromUint64(uint_value->NativeValue()); } - if (auto double_value = As(value); double_value) { + if (auto double_value = value.AsDouble(); double_value) { return CelNumber::FromDouble(double_value->NativeValue()); } return absl::InvalidArgumentError( diff --git a/extensions/protobuf/BUILD b/extensions/protobuf/BUILD index 002da4764..eae7a7b68 100644 --- a/extensions/protobuf/BUILD +++ b/extensions/protobuf/BUILD @@ -118,7 +118,6 @@ cc_test( deps = [ ":value", "//base:attributes", - "//common:casting", "//common:value", "//common:value_kind", "//common:value_testing", @@ -166,7 +165,6 @@ cc_library( srcs = ["bind_proto_to_activation.cc"], hdrs = ["bind_proto_to_activation.h"], deps = [ - "//common:casting", "//common:value", "//internal:status_macros", "//runtime:activation", @@ -184,14 +182,12 @@ cc_test( srcs = ["bind_proto_to_activation_test.cc"], deps = [ ":bind_proto_to_activation", - "//common:casting", "//common:value", "//common:value_testing", "//internal:testing", "//runtime:activation", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:wrappers_cc_proto", diff --git a/extensions/protobuf/bind_proto_to_activation_test.cc b/extensions/protobuf/bind_proto_to_activation_test.cc index 680b4b353..a0ab69123 100644 --- a/extensions/protobuf/bind_proto_to_activation_test.cc +++ b/extensions/protobuf/bind_proto_to_activation_test.cc @@ -17,8 +17,6 @@ #include "google/protobuf/wrappers.pb.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" -#include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "internal/testing.h" @@ -123,7 +121,7 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationDefaultAny) { MATCHER_P(IsListValueOfSize, size, "") { const Value& v = arg; - auto value = As(v); + auto value = v.AsList(); if (!value) { return false; } @@ -185,7 +183,7 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeatedComplex) { MATCHER_P(IsMapValueOfSize, size, "") { const Value& v = arg; - auto value = As(v); + auto value = v.AsMap(); if (!value) { return false; } diff --git a/extensions/protobuf/value_test.cc b/extensions/protobuf/value_test.cc index b43c65968..712745c36 100644 --- a/extensions/protobuf/value_test.cc +++ b/extensions/protobuf/value_test.cc @@ -32,7 +32,6 @@ #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "base/attribute.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_kind.h" #include "common/value_testing.h" @@ -267,7 +266,7 @@ TEST_F(ProtoValueWrapTest, GetFieldNoSuchField) { descriptor_pool(), message_factory(), arena())); ASSERT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); EXPECT_THAT(struct_value.GetFieldByName("does_not_exist", descriptor_pool(), message_factory(), arena()), IsOkAndHolds(ErrorValueIs(StatusIs(absl::StatusCode::kNotFound, @@ -289,7 +288,7 @@ TEST_F(ProtoValueWrapTest, GetFieldByNumber) { single_bytes: "foo")pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); EXPECT_THAT(struct_value.GetFieldByNumber( TestAllTypes::kSingleInt32FieldNumber, descriptor_pool(), @@ -349,7 +348,7 @@ TEST_F(ProtoValueWrapTest, GetFieldByNumberNoSuchField) { single_bytes: "foo")pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); EXPECT_THAT(struct_value.GetFieldByNumber(999, descriptor_pool(), message_factory(), arena()), @@ -370,7 +369,7 @@ TEST_F(ProtoValueWrapTest, HasFieldByNumber) { single_int64: 2)pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); EXPECT_THAT( struct_value.HasFieldByNumber(TestAllTypes::kSingleInt32FieldNumber), @@ -393,7 +392,7 @@ TEST_F(ProtoValueWrapTest, HasFieldByNumberNoSuchField) { single_int64: 2)pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); // Has returns a status directly instead of a CEL error as in Get. EXPECT_THAT( @@ -450,7 +449,7 @@ TEST_F(ProtoValueWrapTest, ProtoMessageForEachField) { )pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); std::vector fields; auto cb = [&fields](absl::string_view field, @@ -473,7 +472,7 @@ TEST_F(ProtoValueWrapTest, ProtoMessageQualify) { )pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); std::vector qualifiers{ FieldSpecifier{TestAllTypes::kStandaloneMessageFieldNumber, @@ -500,7 +499,7 @@ TEST_F(ProtoValueWrapTest, ProtoMessageQualifyHas) { )pb"), descriptor_pool(), message_factory(), arena())); EXPECT_THAT(value, StructValueIs(_)); - StructValue struct_value = Cast(value); + StructValue struct_value = value.GetStruct(); std::vector qualifiers{ FieldSpecifier{TestAllTypes::kStandaloneMessageFieldNumber, @@ -525,15 +524,15 @@ TEST_F(ProtoValueWrapTest, ProtoInt64MapListKeys) { R"pb( map_int64_int64 { key: 10 value: 20 })pb"), descriptor_pool(), message_factory(), arena())); - ASSERT_OK_AND_ASSIGN(auto map_value, Cast(value).GetFieldByName( + ASSERT_OK_AND_ASSIGN(auto map_value, value.GetStruct().GetFieldByName( "map_int64_int64", descriptor_pool(), message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -550,15 +549,15 @@ TEST_F(ProtoValueWrapTest, ProtoInt32MapListKeys) { R"pb( map_int32_int64 { key: 10 value: 20 })pb"), descriptor_pool(), message_factory(), arena())); - ASSERT_OK_AND_ASSIGN(auto map_value, Cast(value).GetFieldByName( + ASSERT_OK_AND_ASSIGN(auto map_value, value.GetStruct().GetFieldByName( "map_int32_int64", descriptor_pool(), message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -575,15 +574,15 @@ TEST_F(ProtoValueWrapTest, ProtoBoolMapListKeys) { R"pb( map_bool_int64 { key: false value: 20 })pb"), descriptor_pool(), message_factory(), arena())); - ASSERT_OK_AND_ASSIGN(auto map_value, Cast(value).GetFieldByName( + ASSERT_OK_AND_ASSIGN(auto map_value, value.GetStruct().GetFieldByName( "map_bool_int64", descriptor_pool(), message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -602,14 +601,14 @@ TEST_F(ProtoValueWrapTest, ProtoUint32MapListKeys) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto map_value, - Cast(value).GetFieldByName( - "map_uint32_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_uint32_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -628,14 +627,14 @@ TEST_F(ProtoValueWrapTest, ProtoUint64MapListKeys) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto map_value, - Cast(value).GetFieldByName( - "map_uint64_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_uint64_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -655,14 +654,14 @@ TEST_F(ProtoValueWrapTest, ProtoStringMapListKeys) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto map_value, - Cast(value).GetFieldByName( - "map_string_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_string_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(map_value, MapValueIs(_)); ASSERT_OK_AND_ASSIGN(ListValue key_set, - Cast(map_value).ListKeys( - descriptor_pool(), message_factory(), arena())); + map_value.GetMap().ListKeys(descriptor_pool(), + message_factory(), arena())); EXPECT_THAT(key_set.Size(), IsOkAndHolds(1)); @@ -683,12 +682,12 @@ TEST_F(ProtoValueWrapTest, ProtoMapIterator) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto field_value, - Cast(value).GetFieldByName( - "map_int64_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_int64_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(field_value, MapValueIs(_)); - MapValue map_value = Cast(field_value); + MapValue map_value = field_value.GetMap(); std::vector keys; @@ -714,12 +713,12 @@ TEST_F(ProtoValueWrapTest, ProtoMapForEach) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto field_value, - Cast(value).GetFieldByName( - "map_int64_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("map_int64_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(field_value, MapValueIs(_)); - MapValue map_value = Cast(field_value); + MapValue map_value = field_value.GetMap(); std::vector> pairs; @@ -747,12 +746,12 @@ TEST_F(ProtoValueWrapTest, ProtoListIterator) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto field_value, - Cast(value).GetFieldByName( - "repeated_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("repeated_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(field_value, ListValueIs(_)); - ListValue list_value = Cast(field_value); + ListValue list_value = field_value.GetList(); std::vector elements; @@ -777,12 +776,12 @@ TEST_F(ProtoValueWrapTest, ProtoListForEachWithIndex) { descriptor_pool(), message_factory(), arena())); ASSERT_OK_AND_ASSIGN( auto field_value, - Cast(value).GetFieldByName( - "repeated_int64", descriptor_pool(), message_factory(), arena())); + value.GetStruct().GetFieldByName("repeated_int64", descriptor_pool(), + message_factory(), arena())); ASSERT_THAT(field_value, ListValueIs(_)); - ListValue list_value = Cast(field_value); + ListValue list_value = field_value.GetList(); std::vector> elements; diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 8e1a95440..07e0ae4c0 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -39,7 +39,6 @@ #include "base/builtins.h" #include "common/ast.h" #include "common/ast_rewrite.h" -#include "common/casting.h" #include "common/constant.h" #include "common/expr.h" #include "common/function_descriptor.h" @@ -853,7 +852,7 @@ absl::Status RecursiveImpl::Evaluate(ExecutionFrameBase& frame, Value& result, AttributeTrail& attribute) const { CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute)); - if (InstanceOf(result) || InstanceOf(result)) { + if (result.IsError() || result.IsUnknown()) { // Just forward. return absl::OkStatus(); } @@ -868,12 +867,11 @@ absl::Status RecursiveImpl::Evaluate(ExecutionFrameBase& frame, Value& result, } } - if (!InstanceOf(result)) { + if (!result.IsStruct()) { return absl::InvalidArgumentError( "Expected struct type for select optimization"); } - CEL_ASSIGN_OR_RETURN(result, - impl_.ApplySelect(frame, Cast(result))); + CEL_ASSIGN_OR_RETURN(result, impl_.ApplySelect(frame, result.GetStruct())); return absl::OkStatus(); } diff --git a/runtime/BUILD b/runtime/BUILD index 57bdb1a63..84cf40e75 100644 --- a/runtime/BUILD +++ b/runtime/BUILD @@ -133,13 +133,11 @@ cc_test( deps = [ ":activation", ":bind_proto_to_activation", - "//common:casting", "//common:value", "//common:value_testing", "//internal:testing", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/types:optional", "@com_google_cel_spec//proto/cel/expr/conformance/proto2:test_all_types_cc_proto", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:wrappers_cc_proto", @@ -580,7 +578,6 @@ cc_library( ":runtime_builder", ":runtime_options", "//base:function_adapter", - "//common:casting", "//common:type", "//common:value", "//internal:casts", diff --git a/runtime/bind_proto_to_activation_test.cc b/runtime/bind_proto_to_activation_test.cc index e7935b92a..e28fbef1b 100644 --- a/runtime/bind_proto_to_activation_test.cc +++ b/runtime/bind_proto_to_activation_test.cc @@ -14,11 +14,11 @@ #include "runtime/bind_proto_to_activation.h" +#include + #include "google/protobuf/wrappers.pb.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" -#include "absl/types/optional.h" -#include "common/casting.h" #include "common/value.h" #include "common/value_testing.h" #include "internal/testing.h" @@ -122,7 +122,7 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationDefaultAny) { MATCHER_P(IsListValueOfSize, size, "") { const Value& v = arg; - auto value = As(v); + auto value = v.AsList(); if (!value) { return false; } @@ -184,7 +184,7 @@ TEST_F(BindProtoToActivationTest, BindProtoToActivationRepeatedComplex) { MATCHER_P(IsMapValueOfSize, size, "") { const Value& v = arg; - auto value = As(v); + auto value = v.AsMap(); if (!value) { return false; } diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index c5475a024..28703ef3b 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -138,7 +138,6 @@ cc_library( "function_adapter.h", ], deps = [ - "//common:casting", "//common:kind", "//common:value", "@com_google_absl//absl/status", diff --git a/runtime/internal/function_adapter.h b/runtime/internal/function_adapter.h index 1ff497486..3a0ed1beb 100644 --- a/runtime/internal/function_adapter.h +++ b/runtime/internal/function_adapter.h @@ -24,7 +24,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/time/time.h" -#include "common/casting.h" #include "common/kind.h" #include "common/value.h" @@ -162,24 +161,24 @@ struct ValueToAdaptedVisitor { template absl::Status operator()(T* out) const { - if (!InstanceOf>(input)) { + if (!input.Is>()) { return absl::InvalidArgumentError( absl::StrCat("expected ", ValueKindToString(T::kKind), " value")); } - *out = Cast>(input); + *out = input.Get>(); return absl::OkStatus(); } template absl::Status operator()(T** out) const { - if (!InstanceOf>(input)) { + if (!input.Is>()) { return absl::InvalidArgumentError( absl::StrCat("expected ", ValueKindToString(T::kKind), " value")); } static_assert(std::is_lvalue_reference_v< - decltype(Cast>(input))>, + decltype(input.Get>())>, "expected l-value reference return type for Cast."); - *out = &Cast>(input); + *out = &input.Get>(); return absl::OkStatus(); } diff --git a/runtime/optional_types.cc b/runtime/optional_types.cc index 6e5dbc12b..91e5b0543 100644 --- a/runtime/optional_types.cc +++ b/runtime/optional_types.cc @@ -27,7 +27,6 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "base/function_adapter.h" -#include "common/casting.h" #include "common/type.h" #include "common/value.h" #include "internal/casts.h" @@ -148,7 +147,7 @@ absl::StatusOr MapOptIndexOptionalValue( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { absl::optional value; - if (auto double_key = cel::As(key); double_key) { + if (auto double_key = key.AsDouble(); double_key) { // Try int/uint. auto number = internal::Number::FromDouble(double_key->NativeValue()); if (number.LosslessConvertibleToInt()) { @@ -217,17 +216,17 @@ absl::StatusOr OptionalOptIndexOptionalValue( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) { - if (auto optional_value = As(opaque_value); optional_value) { + if (auto optional_value = opaque_value.AsOptional(); optional_value) { if (!optional_value->HasValue()) { return OptionalValue::None(); } auto container = optional_value->Value(); - if (auto map_value = cel::As(container); map_value) { + if (auto map_value = container.AsMap(); map_value) { return MapOptIndexOptionalValue(*map_value, key, descriptor_pool, message_factory, arena); } - if (auto list_value = cel::As(container); list_value) { - if (auto int_value = cel::As(key); int_value) { + if (auto list_value = container.AsList(); list_value) { + if (auto int_value = key.AsInt(); int_value) { return ListOptIndexOptionalInt(*list_value, int_value->NativeValue(), descriptor_pool, message_factory, arena); } diff --git a/runtime/standard/container_membership_functions.cc b/runtime/standard/container_membership_functions.cc index cc0638429..26c115103 100644 --- a/runtime/standard/container_membership_functions.cc +++ b/runtime/standard/container_membership_functions.cc @@ -16,7 +16,6 @@ #include #include -#include #include "absl/base/nullability.h" #include "absl/status/status.h" @@ -50,7 +49,7 @@ bool ValueEquals(const Value& value, T other); template <> bool ValueEquals(const Value& value, bool other) { - if (auto bool_value = As(value); bool_value) { + if (auto bool_value = value.As(); bool_value) { return bool_value->NativeValue() == other; } return false; @@ -58,7 +57,7 @@ bool ValueEquals(const Value& value, bool other) { template <> bool ValueEquals(const Value& value, int64_t other) { - if (auto int_value = As(value); int_value) { + if (auto int_value = value.As(); int_value) { return int_value->NativeValue() == other; } return false; @@ -66,7 +65,7 @@ bool ValueEquals(const Value& value, int64_t other) { template <> bool ValueEquals(const Value& value, uint64_t other) { - if (auto uint_value = As(value); uint_value) { + if (auto uint_value = value.As(); uint_value) { return uint_value->NativeValue() == other; } return false; @@ -74,7 +73,7 @@ bool ValueEquals(const Value& value, uint64_t other) { template <> bool ValueEquals(const Value& value, double other) { - if (auto double_value = As(value); double_value) { + if (auto double_value = value.As(); double_value) { return double_value->NativeValue() == other; } return false; @@ -82,7 +81,7 @@ bool ValueEquals(const Value& value, double other) { template <> bool ValueEquals(const Value& value, const StringValue& other) { - if (auto string_value = As(value); string_value) { + if (auto string_value = value.As(); string_value) { return string_value->Equals(other); } return false; @@ -90,7 +89,7 @@ bool ValueEquals(const Value& value, const StringValue& other) { template <> bool ValueEquals(const Value& value, const BytesValue& other) { - if (auto bytes_value = As(value); bytes_value) { + if (auto bytes_value = value.As(); bytes_value) { return bytes_value->Equals(other); } return false; From 72ec7f6a7439ba1908cefb9a925116c4cb326ea0 Mon Sep 17 00:00:00 2001 From: Justin King Date: Wed, 7 Oct 2026 14:54:25 -0700 Subject: [PATCH 271/350] Fix bug in `cel::common_internal::ByteString::TryFlat` PiperOrigin-RevId: 995373516 --- common/internal/byte_string.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/internal/byte_string.cc b/common/internal/byte_string.cc index 468bf893a..2b713b7b1 100644 --- a/common/internal/byte_string.cc +++ b/common/internal/byte_string.cc @@ -253,7 +253,7 @@ absl::optional ByteString::TryFlat() const { return GetMedium(); case ByteStringKind::kLarge: { if (auto flat = rep_.large.data->TryFlat(); flat.has_value()) { - return flat->substr(rep_.large.offset, rep_.large.offset); + return flat->substr(rep_.large.offset, rep_.large.size); } return absl::nullopt; } From f393ee4258b982770eecf55f9c6a70fb2c6cad7f Mon Sep 17 00:00:00 2001 From: Justin King Date: Wed, 7 Oct 2026 15:02:03 -0700 Subject: [PATCH 272/350] Move trivial `Value::As*` and `Value::Get*` accessors into the header PiperOrigin-RevId: 995378297 --- common/value.cc | 490 ----------------------------------------------- common/value.h | 493 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 493 insertions(+), 490 deletions(-) diff --git a/common/value.cc b/common/value.cc index fa937bdf0..bc8a3d7a0 100644 --- a/common/value.cc +++ b/common/value.cc @@ -43,7 +43,6 @@ #include "absl/types/source_location.h" #include "absl/types/variant.h" #include "common/allocator.h" -#include "common/optional_ref.h" #include "common/type.h" #include "common/value_kind.h" #include "common/values/list_value_builder.h" @@ -2007,60 +2006,6 @@ Value Value::WrapMapFieldValueUnsafe( message_factory, arena); } -optional_ref Value::AsBytes() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsBytes() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -absl::optional Value::AsDouble() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsDuration() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -optional_ref Value::AsError() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsError() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -absl::optional Value::AsInt() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - absl::optional Value::AsList() const& { if (const auto* alternative = variant_.As(); alternative != nullptr) { @@ -2141,205 +2086,6 @@ absl::optional Value::AsMap() && { return std::nullopt; } -absl::optional Value::AsMessage() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsMessage() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -absl::optional Value::AsNull() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -optional_ref Value::AsOpaque() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsOpaque() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsOptional() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr && alternative->IsOptional()) { - return static_cast(*alternative); - } - return std::nullopt; -} - -absl::optional Value::AsOptional() && { - if (auto* alternative = variant_.As(); - alternative != nullptr && alternative->IsOptional()) { - return static_cast(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsParsedJsonList() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsParsedJsonList() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsParsedJsonMap() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsParsedJsonMap() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsCustomList() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsCustomList() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsCustomMap() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsCustomMap() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsParsedMapField() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsParsedMapField() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsParsedMessage() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsParsedMessage() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsParsedRepeatedField() - const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsParsedRepeatedField() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsCustomStruct() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsCustomStruct() && { - if (auto* alternative = variant_.As(); - alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -optional_ref Value::AsString() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsString() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - absl::optional Value::AsStruct() const& { if (const auto* alternative = variant_.As(); @@ -2373,87 +2119,6 @@ absl::optional Value::AsStruct() && { return std::nullopt; } -absl::optional Value::AsTimestamp() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -optional_ref Value::AsType() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsType() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -absl::optional Value::AsUint() const { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -optional_ref Value::AsUnknown() const& { - if (const auto* alternative = variant_.As(); - alternative != nullptr) { - return *alternative; - } - return std::nullopt; -} - -absl::optional Value::AsUnknown() && { - if (auto* alternative = variant_.As(); alternative != nullptr) { - return std::move(*alternative); - } - return std::nullopt; -} - -const BytesValue& Value::GetBytes() const& { - ABSL_DCHECK(IsBytes()) << *this; - return variant_.Get(); -} - -BytesValue Value::GetBytes() && { - ABSL_DCHECK(IsBytes()) << *this; - return std::move(variant_).Get(); -} - -DoubleValue Value::GetDouble() const { - ABSL_DCHECK(IsDouble()) << *this; - return variant_.Get(); -} - -DurationValue Value::GetDuration() const { - ABSL_DCHECK(IsDuration()) << *this; - return variant_.Get(); -} - -const ErrorValue& Value::GetError() const& { - ABSL_DCHECK(IsError()) << *this; - return variant_.Get(); -} - -ErrorValue Value::GetError() && { - ABSL_DCHECK(IsError()) << *this; - return std::move(variant_).Get(); -} - -IntValue Value::GetInt() const { - ABSL_DCHECK(IsInt()) << *this; - return variant_.Get(); -} - #ifdef ABSL_HAVE_EXCEPTIONS #define CEL_VALUE_THROW_BAD_VARIANT_ACCESS() throw absl::bad_variant_access() #else @@ -2545,131 +2210,6 @@ MapValue Value::GetMap() && { CEL_VALUE_THROW_BAD_VARIANT_ACCESS(); } -MessageValue Value::GetMessage() const& { - ABSL_DCHECK(IsMessage()) << *this; - return variant_.Get(); -} - -MessageValue Value::GetMessage() && { - ABSL_DCHECK(IsMessage()) << *this; - return std::move(variant_).Get(); -} - -NullValue Value::GetNull() const { - ABSL_DCHECK(IsNull()) << *this; - return variant_.Get(); -} - -const OpaqueValue& Value::GetOpaque() const& { - ABSL_DCHECK(IsOpaque()) << *this; - return variant_.Get(); -} - -OpaqueValue Value::GetOpaque() && { - ABSL_DCHECK(IsOpaque()) << *this; - return std::move(variant_).Get(); -} - -const OptionalValue& Value::GetOptional() const& { - ABSL_DCHECK(IsOptional()) << *this; - return static_cast(variant_.Get()); -} - -OptionalValue Value::GetOptional() && { - ABSL_DCHECK(IsOptional()) << *this; - return static_cast(std::move(variant_).Get()); -} - -const ParsedJsonListValue& Value::GetParsedJsonList() const& { - ABSL_DCHECK(IsParsedJsonList()) << *this; - return variant_.Get(); -} - -ParsedJsonListValue Value::GetParsedJsonList() && { - ABSL_DCHECK(IsParsedJsonList()) << *this; - return std::move(variant_).Get(); -} - -const ParsedJsonMapValue& Value::GetParsedJsonMap() const& { - ABSL_DCHECK(IsParsedJsonMap()) << *this; - return variant_.Get(); -} - -ParsedJsonMapValue Value::GetParsedJsonMap() && { - ABSL_DCHECK(IsParsedJsonMap()) << *this; - return std::move(variant_).Get(); -} - -const CustomListValue& Value::GetCustomList() const& { - ABSL_DCHECK(IsCustomList()) << *this; - return variant_.Get(); -} - -CustomListValue Value::GetCustomList() && { - ABSL_DCHECK(IsCustomList()) << *this; - return std::move(variant_).Get(); -} - -const CustomMapValue& Value::GetCustomMap() const& { - ABSL_DCHECK(IsCustomMap()) << *this; - return variant_.Get(); -} - -CustomMapValue Value::GetCustomMap() && { - ABSL_DCHECK(IsCustomMap()) << *this; - return std::move(variant_).Get(); -} - -const ParsedMapFieldValue& Value::GetParsedMapField() const& { - ABSL_DCHECK(IsParsedMapField()) << *this; - return variant_.Get(); -} - -ParsedMapFieldValue Value::GetParsedMapField() && { - ABSL_DCHECK(IsParsedMapField()) << *this; - return std::move(variant_).Get(); -} - -const ParsedMessageValue& Value::GetParsedMessage() const& { - ABSL_DCHECK(IsParsedMessage()) << *this; - return variant_.Get(); -} - -ParsedMessageValue Value::GetParsedMessage() && { - ABSL_DCHECK(IsParsedMessage()) << *this; - return std::move(variant_).Get(); -} - -const ParsedRepeatedFieldValue& Value::GetParsedRepeatedField() const& { - ABSL_DCHECK(IsParsedRepeatedField()) << *this; - return variant_.Get(); -} - -ParsedRepeatedFieldValue Value::GetParsedRepeatedField() && { - ABSL_DCHECK(IsParsedRepeatedField()) << *this; - return std::move(variant_).Get(); -} - -const CustomStructValue& Value::GetCustomStruct() const& { - ABSL_DCHECK(IsCustomStruct()) << *this; - return variant_.Get(); -} - -CustomStructValue Value::GetCustomStruct() && { - ABSL_DCHECK(IsCustomStruct()) << *this; - return std::move(variant_).Get(); -} - -const StringValue& Value::GetString() const& { - ABSL_DCHECK(IsString()) << *this; - return variant_.Get(); -} - -StringValue Value::GetString() && { - ABSL_DCHECK(IsString()) << *this; - return std::move(variant_).Get(); -} - StructValue Value::GetStruct() const& { ABSL_DCHECK(IsStruct()) << *this; if (const auto* alternative = @@ -2705,36 +2245,6 @@ StructValue Value::GetStruct() && { CEL_VALUE_THROW_BAD_VARIANT_ACCESS(); } -TimestampValue Value::GetTimestamp() const { - ABSL_DCHECK(IsTimestamp()) << *this; - return variant_.Get(); -} - -const TypeValue& Value::GetType() const& { - ABSL_DCHECK(IsType()) << *this; - return variant_.Get(); -} - -TypeValue Value::GetType() && { - ABSL_DCHECK(IsType()) << *this; - return std::move(variant_).Get(); -} - -UintValue Value::GetUint() const { - ABSL_DCHECK(IsUint()) << *this; - return variant_.Get(); -} - -const UnknownValue& Value::GetUnknown() const& { - ABSL_DCHECK(IsUnknown()) << *this; - return variant_.Get(); -} - -UnknownValue Value::GetUnknown() && { - ABSL_DCHECK(IsUnknown()) << *this; - return std::move(variant_).Get(); -} - namespace { class EmptyValueIterator final : public ValueIterator { diff --git a/common/value.h b/common/value.h index c2c8a0c7a..9278cf4de 100644 --- a/common/value.h +++ b/common/value.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -2946,6 +2947,498 @@ absl::StatusOr RepeatedFieldAccessorFor( inline BytesValue::BytesValue(const StringValue& other) : BytesValue(other.value_) {} +inline optional_ref Value::AsBytes() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsBytes() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsDouble() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsDuration() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsError() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsError() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsInt() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsMessage() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsMessage() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsNull() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsOpaque() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsOpaque() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsOptional() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr && alternative->IsOptional()) { + return static_cast(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsOptional() && { + if (auto* alternative = variant_.As(); + alternative != nullptr && alternative->IsOptional()) { + return static_cast(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsParsedJsonList() + const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsParsedJsonList() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsParsedJsonMap() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsParsedJsonMap() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsCustomList() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsCustomList() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsCustomMap() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsCustomMap() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsParsedMapField() + const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsParsedMapField() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsParsedMessage() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsParsedMessage() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref +Value::AsParsedRepeatedField() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional +Value::AsParsedRepeatedField() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsCustomStruct() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsCustomStruct() && { + if (auto* alternative = variant_.As(); + alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline optional_ref Value::AsString() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsString() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsTimestamp() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsType() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsType() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline absl::optional Value::AsUint() const { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline optional_ref Value::AsUnknown() const& { + if (const auto* alternative = variant_.As(); + alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + +inline absl::optional Value::AsUnknown() && { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return std::move(*alternative); + } + return std::nullopt; +} + +inline const BytesValue& Value::GetBytes() const& { + ABSL_DCHECK(IsBytes()) << *this; + return variant_.Get(); +} + +inline BytesValue Value::GetBytes() && { + ABSL_DCHECK(IsBytes()) << *this; + return std::move(variant_).Get(); +} + +inline DoubleValue Value::GetDouble() const { + ABSL_DCHECK(IsDouble()) << *this; + return variant_.Get(); +} + +inline DurationValue Value::GetDuration() const { + ABSL_DCHECK(IsDuration()) << *this; + return variant_.Get(); +} + +inline const ErrorValue& Value::GetError() const& { + ABSL_DCHECK(IsError()) << *this; + return variant_.Get(); +} + +inline ErrorValue Value::GetError() && { + ABSL_DCHECK(IsError()) << *this; + return std::move(variant_).Get(); +} + +inline IntValue Value::GetInt() const { + ABSL_DCHECK(IsInt()) << *this; + return variant_.Get(); +} + +inline MessageValue Value::GetMessage() const& { + ABSL_DCHECK(IsMessage()) << *this; + return variant_.Get(); +} + +inline MessageValue Value::GetMessage() && { + ABSL_DCHECK(IsMessage()) << *this; + return std::move(variant_).Get(); +} + +inline NullValue Value::GetNull() const { + ABSL_DCHECK(IsNull()) << *this; + return variant_.Get(); +} + +inline const OpaqueValue& Value::GetOpaque() const& { + ABSL_DCHECK(IsOpaque()) << *this; + return variant_.Get(); +} + +inline OpaqueValue Value::GetOpaque() && { + ABSL_DCHECK(IsOpaque()) << *this; + return std::move(variant_).Get(); +} + +inline const OptionalValue& Value::GetOptional() const& { + ABSL_DCHECK(IsOptional()) << *this; + return static_cast(variant_.Get()); +} + +inline OptionalValue Value::GetOptional() && { + ABSL_DCHECK(IsOptional()) << *this; + return static_cast(std::move(variant_).Get()); +} + +inline const ParsedJsonListValue& Value::GetParsedJsonList() const& { + ABSL_DCHECK(IsParsedJsonList()) << *this; + return variant_.Get(); +} + +inline ParsedJsonListValue Value::GetParsedJsonList() && { + ABSL_DCHECK(IsParsedJsonList()) << *this; + return std::move(variant_).Get(); +} + +inline const ParsedJsonMapValue& Value::GetParsedJsonMap() const& { + ABSL_DCHECK(IsParsedJsonMap()) << *this; + return variant_.Get(); +} + +inline ParsedJsonMapValue Value::GetParsedJsonMap() && { + ABSL_DCHECK(IsParsedJsonMap()) << *this; + return std::move(variant_).Get(); +} + +inline const CustomListValue& Value::GetCustomList() const& { + ABSL_DCHECK(IsCustomList()) << *this; + return variant_.Get(); +} + +inline CustomListValue Value::GetCustomList() && { + ABSL_DCHECK(IsCustomList()) << *this; + return std::move(variant_).Get(); +} + +inline const CustomMapValue& Value::GetCustomMap() const& { + ABSL_DCHECK(IsCustomMap()) << *this; + return variant_.Get(); +} + +inline CustomMapValue Value::GetCustomMap() && { + ABSL_DCHECK(IsCustomMap()) << *this; + return std::move(variant_).Get(); +} + +inline const ParsedMapFieldValue& Value::GetParsedMapField() const& { + ABSL_DCHECK(IsParsedMapField()) << *this; + return variant_.Get(); +} + +inline ParsedMapFieldValue Value::GetParsedMapField() && { + ABSL_DCHECK(IsParsedMapField()) << *this; + return std::move(variant_).Get(); +} + +inline const ParsedMessageValue& Value::GetParsedMessage() const& { + ABSL_DCHECK(IsParsedMessage()) << *this; + return variant_.Get(); +} + +inline ParsedMessageValue Value::GetParsedMessage() && { + ABSL_DCHECK(IsParsedMessage()) << *this; + return std::move(variant_).Get(); +} + +inline const ParsedRepeatedFieldValue& Value::GetParsedRepeatedField() const& { + ABSL_DCHECK(IsParsedRepeatedField()) << *this; + return variant_.Get(); +} + +inline ParsedRepeatedFieldValue Value::GetParsedRepeatedField() && { + ABSL_DCHECK(IsParsedRepeatedField()) << *this; + return std::move(variant_).Get(); +} + +inline const CustomStructValue& Value::GetCustomStruct() const& { + ABSL_DCHECK(IsCustomStruct()) << *this; + return variant_.Get(); +} + +inline CustomStructValue Value::GetCustomStruct() && { + ABSL_DCHECK(IsCustomStruct()) << *this; + return std::move(variant_).Get(); +} + +inline const StringValue& Value::GetString() const& { + ABSL_DCHECK(IsString()) << *this; + return variant_.Get(); +} + +inline StringValue Value::GetString() && { + ABSL_DCHECK(IsString()) << *this; + return std::move(variant_).Get(); +} + +inline TimestampValue Value::GetTimestamp() const { + ABSL_DCHECK(IsTimestamp()) << *this; + return variant_.Get(); +} + +inline const TypeValue& Value::GetType() const& { + ABSL_DCHECK(IsType()) << *this; + return variant_.Get(); +} + +inline TypeValue Value::GetType() && { + ABSL_DCHECK(IsType()) << *this; + return std::move(variant_).Get(); +} + +inline UintValue Value::GetUint() const { + ABSL_DCHECK(IsUint()) << *this; + return variant_.Get(); +} + +inline const UnknownValue& Value::GetUnknown() const& { + ABSL_DCHECK(IsUnknown()) << *this; + return variant_.Get(); +} + +inline UnknownValue Value::GetUnknown() && { + ABSL_DCHECK(IsUnknown()) << *this; + return std::move(variant_).Get(); +} + } // namespace cel #pragma pop_macro("GetMessage") From 68f523b2d3a51f0c60da6bf471b5b615618dead6 Mon Sep 17 00:00:00 2001 From: Justin King Date: Thu, 8 Oct 2026 11:52:46 -0700 Subject: [PATCH 273/350] Introduce `AttributeMatcher` PiperOrigin-RevId: 995978258 --- base/BUILD | 33 +++ base/attribute.cc | 51 ++++ base/attribute.h | 40 ++- base/attribute_matcher.cc | 366 ++++++++++++++++++++++++++++ base/attribute_matcher.h | 327 +++++++++++++++++++++++++ base/attribute_matcher_test.cc | 427 +++++++++++++++++++++++++++++++++ base/attribute_test.cc | 19 ++ 7 files changed, 1259 insertions(+), 4 deletions(-) create mode 100644 base/attribute_matcher.cc create mode 100644 base/attribute_matcher.h create mode 100644 base/attribute_matcher_test.cc diff --git a/base/BUILD b/base/BUILD index 739fc96de..f4b5fcfd5 100644 --- a/base/BUILD +++ b/base/BUILD @@ -34,6 +34,8 @@ cc_library( deps = [ ":kind", "//internal:status_macros", + "//internal:strings", + "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/base:nullability", @@ -59,6 +61,37 @@ cc_test( ], ) +cc_library( + name = "attribute_matcher", + srcs = ["attribute_matcher.cc"], + hdrs = ["attribute_matcher.h"], + deps = [ + ":attributes", + "//internal:status_macros", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:source_location", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "attribute_matcher_test", + srcs = ["attribute_matcher_test.cc"], + deps = [ + ":attribute_matcher", + ":attributes", + "//internal:testing", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + ], +) + cc_library( name = "kind", hdrs = ["kind.h"], diff --git a/base/attribute.cc b/base/attribute.cc index 97fa75fa7..768e7c4f4 100644 --- a/base/attribute.cc +++ b/base/attribute.cc @@ -20,6 +20,7 @@ #include #include +#include "absl/algorithm/container.h" #include "absl/base/macros.h" #include "absl/base/nullability.h" #include "absl/log/absl_check.h" @@ -30,6 +31,7 @@ #include "absl/strings/string_view.h" #include "base/kind.h" #include "internal/status_macros.h" +#include "internal/strings.h" namespace cel { @@ -119,6 +121,26 @@ struct AttributeQualifierTypeVisitor final { Kind operator()(bool) const { return Kind::kBool; } }; +struct AttributeQualifierToString { + void operator()(std::monostate) const {} + + void operator()(bool value) const { output.append(value ? "true" : "false"); } + + void operator()(int64_t value) const { absl::StrAppend(&output, value); } + + void operator()(uint64_t value) const { + absl::StrAppend(&output, value, "u"); + } + + void operator()(absl::string_view value) const { + absl::StrAppend(&output, internal::FormatStringLiteral(value)); + } + + void operator()(common_internal::WildcardType) const { output.append("*"); } + + std::string& output; +}; + } // namespace Kind AttributeQualifier::kind() const { @@ -193,6 +215,30 @@ absl::StatusOr Attribute::AsString() const { return result; } +std::string Attribute::ToString() const { + std::string result; + result.append(variable_name()); + for (const auto& qualifier : qualifier_path()) { + result.push_back('['); + std::visit(AttributeQualifierToString{result}, + common_internal::AsVariant(qualifier)); + result.push_back(']'); + } + return result; +} + +std::string AttributePattern::ToString() const { + std::string result; + result.append(variable()); + for (const auto& qualifier : qualifier_path()) { + result.push_back('['); + std::visit(AttributeQualifierToString{result}, + common_internal::AsVariant(qualifier)); + result.push_back(']'); + } + return result; +} + std::string AttributeQualifier::ToString() const { std::string result; absl::Status status = @@ -517,4 +563,9 @@ bool operator<(const AttributeQualifierPattern& lhs, common_internal::AsVariant(rhs)); } +bool operator==(const AttributePattern& lhs, const AttributePattern& rhs) { + return lhs.variable() == rhs.variable() && + absl::c_equal(lhs.qualifier_path(), rhs.qualifier_path()); +} + } // namespace cel diff --git a/base/attribute.h b/base/attribute.h index 2d589368d..c4439a176 100644 --- a/base/attribute.h +++ b/base/attribute.h @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -107,10 +108,8 @@ class AttributeQualifier { } AttributeQualifier() = default; - AttributeQualifier(const AttributeQualifier&) = default; AttributeQualifier(AttributeQualifier&&) = default; - AttributeQualifier& operator=(const AttributeQualifier&) = default; AttributeQualifier& operator=(AttributeQualifier&&) = default; @@ -446,6 +445,13 @@ class AttributeQualifierPattern { return AttributeQualifierPattern(std::in_place_type); } + AttributeQualifierPattern() = default; + AttributeQualifierPattern(const AttributeQualifierPattern&) = default; + AttributeQualifierPattern(AttributeQualifierPattern&&) = default; + AttributeQualifierPattern& operator=(const AttributeQualifierPattern&) = + default; + AttributeQualifierPattern& operator=(AttributeQualifierPattern&&) = default; + // NOLINTNEXTLINE(google-explicit-constructor) AttributeQualifierPattern(const AttributeQualifier& value) : value_(std::visit( @@ -840,6 +846,9 @@ class Attribute { absl::StatusOr AsString() const; + [[nodiscard]] + std::string ToString() const; + private: struct Impl final { Impl(std::string variable_name, @@ -854,6 +863,11 @@ class Attribute { std::shared_ptr impl_; }; +template +void AbslStringify(S& sink, const Attribute& attribute) { + sink.Append(attribute.ToString()); +} + // AttributePattern is a fully-qualified absolute attribute path pattern. // Supported segments steps in the path are: // - field selection; @@ -868,8 +882,9 @@ class AttributePattern { FULL // Pattern matches an attribute itself. }; - AttributePattern(std::string variable, - std::vector qualifier_path) + explicit AttributePattern( + std::string variable, + std::vector qualifier_path = {}) : variable_(std::move(variable)), qualifier_path_(std::move(qualifier_path)) {} @@ -902,11 +917,28 @@ class AttributePattern { return result; } + [[nodiscard]] + std::string ToString() const; + private: std::string variable_; std::vector qualifier_path_; }; +template +void AbslStringify(S& sink, const AttributePattern& pattern) { + sink.Append(pattern.ToString()); +} + +[[nodiscard]] +bool operator==(const AttributePattern& lhs, const AttributePattern& rhs); + +[[nodiscard]] +inline bool operator!=(const AttributePattern& lhs, + const AttributePattern& rhs) { + return !operator==(lhs, rhs); +} + struct FieldSpecifier { int64_t number; std::string name; diff --git a/base/attribute_matcher.cc b/base/attribute_matcher.cc new file mode 100644 index 000000000..328045aa1 --- /dev/null +++ b/base/attribute_matcher.cc @@ -0,0 +1,366 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "base/attribute_matcher.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/no_destructor.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/source_location.h" +#include "absl/types/span.h" +#include "base/attribute.h" +#include "internal/status_macros.h" + +namespace cel { + +namespace common_internal { + +std::pair AttributeMatcherNode::Add( + const AttributeQualifier& pattern, bool prev_emplaced) { + // If this node was not just created and it has no children, it is a full + // match and additional children cannot be added. + if (wildcard_ == nullptr && children_ == nullptr && !prev_emplaced) { + return {nullptr, false}; + } + // If this node has a wildcard already, no more children can be added. + if (wildcard_ != nullptr) { + return {nullptr, false}; + } + if (children_ == nullptr) { + children_ = std::make_unique>(); + } + auto [child, emplaced] = children_->emplace(this, pattern); + return {const_cast(&*child), emplaced}; +} + +std::pair AttributeMatcherNode::Add( + const AttributeQualifierPattern& pattern, bool prev_emplaced) { + if (auto qualifier = pattern.ToQualifier(); qualifier.has_value()) { + return Add(*qualifier, prev_emplaced); + } + // If this node was not just created and it has no children, it is a full + // match and additional children cannot be added. + if (wildcard_ == nullptr && children_ == nullptr && !prev_emplaced) { + return {nullptr, false}; + } + // If this node has children already, no wildcard can be added. + if (children_ != nullptr) { + return {nullptr, false}; + } + if (wildcard_ == nullptr) { + wildcard_ = std::make_unique( + this, AttributeQualifierPattern::Wildcard()); + return {wildcard_.get(), true}; + } + return {wildcard_.get(), false}; +} + +size_t AttributeMatcherNode::RemoveChildren() { + if (children_ == nullptr && wildcard_ == nullptr) { + return 0; + } + return RemoveChildrenInternal(); +} + +size_t AttributeMatcherNode::RemoveChildrenInternal() { + if (children_ == nullptr && wildcard_ == nullptr) { + return 1; + } + size_t count = 0; + if (children_ != nullptr) { + auto children_begin = children_->begin(); + const auto children_end = children_->end(); + while (children_begin != children_end) { + count += const_cast(&*children_begin) + ->RemoveChildrenInternal(); + children_begin = children_->erase(children_begin); + } + children_.reset(); + } + if (wildcard_ != nullptr) { + count += wildcard_->RemoveChildrenInternal(); + wildcard_.reset(); + } + return count; +} + +bool AttributeMatcherNode::Remove() { + AttributeMatcherNode* parent = this->parent(); + if (parent == nullptr) { + return children_ == nullptr && wildcard_ == nullptr; + } + if (children_ == nullptr && wildcard_ == nullptr) { + if (key_.IsWildcard()) { + ABSL_DCHECK_EQ(parent->wildcard_.get(), this); + parent->wildcard_.reset(); + } else { + parent->children_->erase(*this); + if (parent->children_->empty()) { + parent->children_.reset(); + } + } + return parent->ChildRemoved(); + } + return false; +} + +bool AttributeMatcherNode::ChildRemoved() { return Remove(); } + +} // namespace common_internal + +namespace { + +absl::Status AttributePatternNotFound( + absl::string_view pattern, + absl::SourceLocation source_location = absl::SourceLocation::current()) { + return absl::NotFoundError( + absl::StrCat("attribute pattern ", pattern, " not found"), + source_location); +} + +bool IsWildcard(const AttributeQualifierPattern& qualifier) { + return qualifier.IsWildcard(); +} + +bool IsWildcard(const AttributeQualifier&) { return false; } + +template +absl::Status ValidateAttribute(absl::string_view variable, + absl::Span qualifiers, + absl::FunctionRef to_string) { + if (variable.empty()) { + return absl::InvalidArgumentError( + "attribute pattern variable name is required"); + } + for (size_t i = 0; i < qualifiers.size(); ++i) { + if (!qualifiers[i]) { + return absl::InvalidArgumentError("bad attribute qualifier pattern"); + } + if (IsWildcard(qualifiers[i]) && i != qualifiers.size() - 1) { + return absl::InvalidArgumentError( + "wildcard qualifier in attribute pattern must be last"); + } + } + return absl::OkStatus(); +} + +} // namespace + +template +absl::Status AttributeMatcher::InsertAttribute( + absl::string_view variable, absl::Span qualifiers, + absl::FunctionRef to_string) { + CEL_RETURN_IF_ERROR(ValidateAttribute(variable, qualifiers, to_string)); + auto [node, emplaced] = AddVariable(variable); + for (const auto& qualifier : qualifiers) { + std::tie(node, emplaced) = node->Add(qualifier, emplaced); + if (node == nullptr) { + break; + } + } + if (node == nullptr) { + return absl::AlreadyExistsError( + absl::StrCat("attribute pattern which is a superset of ", to_string(), + " already exists")); + } + if (!emplaced) { + return absl::AlreadyExistsError(absl::StrCat( + "attribute pattern covering ", to_string(), " already exists")); + } + ++attributes_; + return absl::OkStatus(); +} + +absl::Status AttributeMatcher::InsertAttribute( + const AttributePattern& pattern) { + return InsertAttribute( + pattern.variable(), pattern.qualifier_path(), + [&pattern]() -> std::string { return pattern.ToString(); }); +} + +absl::Status AttributeMatcher::InsertAttribute(const Attribute& attribute) { + return InsertAttribute( + attribute.variable_name(), attribute.qualifier_path(), + [&attribute]() -> std::string { return attribute.ToString(); }); +} + +template +absl::Status AttributeMatcher::UpsertAttribute( + absl::string_view variable, absl::Span qualifiers, + absl::FunctionRef to_string) { + CEL_RETURN_IF_ERROR(ValidateAttribute(variable, qualifiers, to_string)); + auto [node, emplaced] = AddVariable(variable); + for (const auto& qualifier : qualifiers) { + std::tie(node, emplaced) = node->Add(qualifier, emplaced); + if (node == nullptr) { + // subset + return absl::OkStatus(); + } + } + if (node->children_ == nullptr && node->wildcard_ == nullptr) { + // duplicate. + return absl::OkStatus(); + } + if (!emplaced) { + // superset + attributes_ -= node->RemoveChildren(); + ++attributes_; + } + return absl::OkStatus(); +} + +absl::Status AttributeMatcher::UpsertAttribute( + const AttributePattern& pattern) { + return UpsertAttribute( + pattern.variable(), pattern.qualifier_path(), + [&pattern]() -> std::string { return pattern.ToString(); }); +} + +absl::Status AttributeMatcher::UpsertAttribute(const Attribute& attribute) { + return UpsertAttribute( + attribute.variable_name(), attribute.qualifier_path(), + [&attribute]() -> std::string { return attribute.ToString(); }); +} + +template +absl::Status AttributeMatcher::RemoveAttribute( + absl::string_view variable, absl::Span qualifiers, + absl::FunctionRef to_string) { + CEL_RETURN_IF_ERROR(ValidateAttribute(variable, qualifiers, to_string)); + auto children_it = children_.find(AttributeQualifierView::OfString(variable)); + if (children_it == children_.end()) { + return AttributePatternNotFound(to_string()); + } + Node* variable_node = const_cast(&*children_it); + Node* node = variable_node; + for (const auto& qualifier : qualifiers) { + if (IsWildcard(qualifier)) { + if (node->wildcard_ == nullptr) { + return AttributePatternNotFound(to_string()); + } + node = node->wildcard_.get(); + } else { + if (node->children_ == nullptr) { + return AttributePatternNotFound(to_string()); + } + children_it = node->children_->find(qualifier); + if (children_it == node->children_->end()) { + return AttributePatternNotFound(to_string()); + } + node = const_cast(&*children_it); + } + } + if (node->children_ != nullptr || node->wildcard_ != nullptr) { + return AttributePatternNotFound(to_string()); + } + --attributes_; + if (node->Remove()) { + children_.erase(*variable_node); + } + return absl::OkStatus(); +} + +absl::Status AttributeMatcher::RemoveAttribute( + const AttributePattern& pattern) { + return RemoveAttribute( + pattern.variable(), pattern.qualifier_path(), + [&pattern]() -> std::string { return pattern.ToString(); }); +} + +absl::Status AttributeMatcher::RemoveAttribute(const Attribute& attribute) { + return RemoveAttribute( + attribute.variable_name(), attribute.qualifier_path(), + [&attribute]() -> std::string { return attribute.ToString(); }); +} + +struct AttributeMatcher::AttributeCollector { + std::vector result; + absl::string_view variable; + std::vector qualifiers; + + explicit AttributeCollector(size_t capacity) { result.reserve(capacity); } + + void CollectVariable(absl::string_view variable, + const common_internal::AttributeMatcherNode* node) { + if (node->children_ == nullptr && node->wildcard_ == nullptr) { + result.push_back(AttributePattern(std::string(variable), {})); + return; + } + this->variable = variable; + qualifiers.clear(); + if (node->children_ != nullptr) { + for (const auto& child : *node->children_) { + CollectQualifier(child.key(), &child); + } + } + if (node->wildcard_ != nullptr) { + CollectQualifier(AttributeQualifierPattern::Wildcard(), + node->wildcard_.get()); + } + } + + void CollectQualifier(const AttributeQualifierPattern& qualifier, + const common_internal::AttributeMatcherNode* node) { + qualifiers.emplace_back(qualifier); + if (node->children_ == nullptr && node->wildcard_ == nullptr) { + result.push_back(AttributePattern(std::string(variable), qualifiers)); + } else { + if (node->children_ != nullptr) { + for (const auto& child : *node->children_) { + CollectQualifier(child.key(), &child); + } + } + if (node->wildcard_ != nullptr) { + CollectQualifier(AttributeQualifierPattern::Wildcard(), + node->wildcard_.get()); + } + } + qualifiers.pop_back(); + } +}; + +[[nodiscard]] +std::vector AttributeMatcher::GetAttributes() const { + AttributeCollector collector(attributes_); + for (const auto& child : children_) { + // Guaranteed to be string. + collector.CollectVariable(child.key().GetString(), &child); + } + return std::move(collector.result); +} + +std::pair AttributeMatcher::AddVariable( + absl::string_view variable) { + auto [child, emplaced] = + children_.emplace(nullptr, AttributeQualifierPattern::OfString(variable)); + return {const_cast(&*child), emplaced}; +} + +const AttributeMatcher& EmptyAttributeMatcher() { + static const absl::NoDestructor empty; + return *empty; +} + +} // namespace cel diff --git a/base/attribute_matcher.h b/base/attribute_matcher.h new file mode 100644 index 000000000..69174868e --- /dev/null +++ b/base/attribute_matcher.h @@ -0,0 +1,327 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_MATCHER_H_ +#define THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_MATCHER_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "base/attribute.h" + +namespace cel { + +class AttributeMatcher; +class AttributeMatch; + +namespace common_internal { + +// Node in the tree used by AttributeMatcher, represents either a variable or a +// qualifier. Nodes with no parent are variables and all others are qualifiers. +class AttributeMatcherNode { + public: + struct Less { + using is_transparent = void; + + bool operator()(const AttributeMatcherNode& lhs, + const AttributeMatcherNode& rhs) const { + return lhs.key() < rhs.key(); + } + + bool operator()(const AttributeMatcherNode& lhs, + const AttributeQualifierPattern& rhs) const { + return lhs.key() < rhs; + } + + bool operator()(const AttributeQualifierPattern& lhs, + const AttributeMatcherNode& rhs) const { + return lhs < rhs.key(); + } + + bool operator()(const AttributeMatcherNode& lhs, + const AttributeQualifierView& rhs) const { + return lhs.key() < rhs; + } + + bool operator()(const AttributeQualifierView& lhs, + const AttributeMatcherNode& rhs) const { + return lhs < rhs.key(); + } + + bool operator()(const AttributeMatcherNode& lhs, + const AttributeQualifier& rhs) const { + return lhs.key() < rhs; + } + + bool operator()(const AttributeQualifier& lhs, + const AttributeMatcherNode& rhs) const { + return lhs < rhs.key(); + } + }; + + AttributeMatcherNode(AttributeMatcherNode* parent, + const AttributeQualifierPattern& key) + : parent_(parent), + key_(key), + depth_(parent != nullptr ? parent->depth_ + 1 : 0) {} + + [[nodiscard]] + bool IsPartial() const { + return children_ != nullptr || wildcard_ != nullptr; + } + + [[nodiscard]] + bool IsFull() const { + return !IsPartial(); + } + + [[nodiscard]] + const AttributeQualifierPattern& key() const { + return key_; + } + + [[nodiscard]] + AttributeMatcherNode* parent() const { + return parent_; + } + + [[nodiscard]] + size_t depth() const { + return depth_; + } + + [[nodiscard]] + const AttributeMatcherNode* MatchQualifier( + const cel::AttributeQualifierView& qualifier) const { + if (children_ != nullptr) { + auto it = children_->find(qualifier); + if (it != children_->end()) { + return &*it; + } + } else if (wildcard_ != nullptr) { + return wildcard_.get(); + } + return nullptr; + } + + [[nodiscard]] + std::pair Add(const AttributeQualifier& pattern, + bool prev_emplaced); + [[nodiscard]] + std::pair Add( + const AttributeQualifierPattern& pattern, bool prev_emplaced); + + [[nodiscard]] + bool Remove(); + + [[nodiscard]] + size_t RemoveChildren(); + + private: + friend class cel::AttributeMatcher; + + [[nodiscard]] + bool ChildRemoved(); + + [[nodiscard]] + size_t RemoveChildrenInternal(); + + AttributeMatcherNode* const parent_; + const AttributeQualifierPattern key_; + const size_t depth_; + // Pointer stability is required, so use std::set. + std::unique_ptr> children_; + std::unique_ptr wildcard_; +}; + +AttributeMatch MakeAttributeMatch( + const common_internal::AttributeMatcherNode* node); + +[[nodiscard]] +const AttributeMatcherNode* GetAttributeMatcherNode( + const AttributeMatch& match); + +} // namespace common_internal + +class [[nodiscard]] AttributeMatch { + public: + using Type = AttributePattern::MatchType; + + AttributeMatch() = default; + AttributeMatch(const AttributeMatch&) = default; + AttributeMatch& operator=(const AttributeMatch&) = default; + + [[nodiscard]] + Type GetType() const { + return node_ == nullptr ? Type::NONE + : node_->IsPartial() ? Type::PARTIAL + : Type::FULL; + } + + [[nodiscard]] + bool IsNone() const { + return node_ == nullptr; + } + + [[nodiscard]] + bool IsPartial() const { + return node_ != nullptr && node_->IsPartial(); + } + + [[nodiscard]] + bool IsFull() const { + return node_ != nullptr && node_->IsFull(); + } + + // Attempts to match the next qualifier. This should not be called on full + // matches, as those do not have children by definition. Instead the full + // match should be handled. + AttributeMatch MatchQualifier(const AttributeQualifierView& qualifier) const { + ABSL_DCHECK(!IsFull()); + if (IsNone()) { + return AttributeMatch(); + } + return AttributeMatch(node_->MatchQualifier(qualifier)); + } + + private: + friend AttributeMatch common_internal::MakeAttributeMatch( + const common_internal::AttributeMatcherNode* node); + friend const common_internal::AttributeMatcherNode* + common_internal::GetAttributeMatcherNode(const AttributeMatch& match); + + explicit AttributeMatch(const common_internal::AttributeMatcherNode* node) + : node_(node) {} + + const common_internal::AttributeMatcherNode* node_ = nullptr; +}; + +namespace common_internal { + +inline AttributeMatch MakeAttributeMatch( + const common_internal::AttributeMatcherNode* node) { + return AttributeMatch(node); +} + +[[nodiscard]] +inline const AttributeMatcherNode* GetAttributeMatcherNode( + const AttributeMatch& match) { + return match.node_; +} + +} // namespace common_internal + +// AttributeMatcher allows for efficiently checking for unknown or missing +// attributes at runtime in a scalable manner. AttributeMatcher is +// thread-compatible however it must not be modified during any evaluation where +// it is being used. +class AttributeMatcher { + private: + using Node = common_internal::AttributeMatcherNode; + + public: + AttributeMatcher() = default; + AttributeMatcher(const AttributeMatcher&) = delete; + AttributeMatcher(AttributeMatcher&&) = default; + AttributeMatcher& operator=(const AttributeMatcher&) = delete; + AttributeMatcher& operator=(AttributeMatcher&&) = default; + + // Inserts an attribute to the matcher. Returns an error if the attribute + // or attribute pattern being added overlaps with a previously added + // attribute. + absl::Status InsertAttribute(const AttributePattern& pattern); + absl::Status InsertAttribute(const Attribute& attribute); + + // Inserts or updates the attribute in the matcher. If the attribute being + // added is a superset of others, the subsets will be removed and replaced + // with this attribute. If the attribute being added is a subset of others, + // no action is taken. Otherwise the attribute is inserted. + absl::Status UpsertAttribute(const AttributePattern& pattern); + absl::Status UpsertAttribute(const Attribute& attribute); + + // Removes a previously added unknown attribute from the matcher. + absl::Status RemoveAttribute(const AttributePattern& pattern); + absl::Status RemoveAttribute(const Attribute& attribute); + + // Resets the matcher such that all observable behavior is equivalent to a + // default constructed matcher. + void ClearAttributes() { + children_.clear(); + attributes_ = 0; + } + + AttributeMatch MatchVariable(absl::string_view variable) const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + auto it = children_.find(AttributeQualifierView::OfString(variable)); + if (it != children_.end()) { + return common_internal::MakeAttributeMatch(&*it); + } + return AttributeMatch(); + } + + // Returns an array of attribute patterns corresponding to the + // previously added attribute patterns. The order of the attribute + // patterns in the array is not guaranteed. + [[nodiscard]] + std::vector GetAttributes() const; + + // Tests whether any attribute patterns have been added to this + // matcher. + [[nodiscard]] + bool HasAttributes() const { + return !children_.empty(); + } + + private: + struct AttributeCollector; + + [[nodiscard]] + std::pair AddVariable(absl::string_view variable); + + template + absl::Status InsertAttribute(absl::string_view variable, + absl::Span qualifiers, + absl::FunctionRef to_string); + + template + absl::Status UpsertAttribute(absl::string_view variable, + absl::Span qualifiers, + absl::FunctionRef to_string); + + template + absl::Status RemoveAttribute(absl::string_view variable, + absl::Span qualifiers, + absl::FunctionRef to_string); + + // Pointer stability is required. + std::set children_; + size_t attributes_ = 0; +}; + +[[nodiscard]] +const AttributeMatcher& EmptyAttributeMatcher(); + +} // namespace cel + +#endif // THIRD_PARTY_CEL_CPP_BASE_ATTRIBUTE_MATCHER_H_ diff --git a/base/attribute_matcher_test.cc b/base/attribute_matcher_test.cc new file mode 100644 index 000000000..899f2f03f --- /dev/null +++ b/base/attribute_matcher_test.cc @@ -0,0 +1,427 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "base/attribute_matcher.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "base/attribute.h" +#include "internal/testing.h" + +namespace cel { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::testing::_; +using ::testing::UnorderedElementsAre; + +using MatchType = AttributePattern::MatchType; + +MATCHER_P(AttributeMatchIs, type, "") { + return ::testing::ExplainMatchResult(::testing::Eq(type), arg.GetType(), + result_listener); +} + +using ::testing::IsEmpty; + +TEST(AttributeMatcher, Empty) { + AttributeMatcher matcher; + EXPECT_FALSE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), IsEmpty()); + EXPECT_THAT(matcher.MatchVariable("foo"), AttributeMatchIs(MatchType::NONE)); +} + +TEST(AttributeMatcher, BadAttribute) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(Attribute("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo", {AttributeQualifier()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.InsertAttribute( + AttributePattern("foo", {AttributeQualifierPattern()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.UpsertAttribute(Attribute("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.UpsertAttribute(AttributePattern("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.UpsertAttribute(Attribute("foo", {AttributeQualifier()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.UpsertAttribute( + AttributePattern("foo", {AttributeQualifierPattern()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.RemoveAttribute(Attribute("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.RemoveAttribute(AttributePattern("")), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.RemoveAttribute(Attribute("foo", {AttributeQualifier()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.RemoveAttribute( + AttributePattern("foo", {AttributeQualifierPattern()})), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST(AttributeMatcher, Attribute_OneVariable_ZeroQualifier_Subset) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), IsOk()); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::Wildcard()})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern("foo"))); +} + +TEST(AttributeMatcher, + Unknown_AttributePattern_OneVariable_ZeroQualifier_Subset) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), IsOk()); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::Wildcard()})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern("foo"))); +} + +TEST(AttributeMatcher, Unknown_Attribute_OneVariable_TwoQualifier_Superset) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); +} + +TEST(AttributeMatcher, + Unknown_AttributePattern_OneVariable_TwoQualifier_Superset) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); +} + +TEST(AttributeMatcher, Unknown_Attribute_OneVariable_ZeroQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(Attribute("foo")), IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern("foo"))); + EXPECT_THAT(matcher.MatchVariable("foo"), AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_Attribute_OneVariable_OneQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")}))); + EXPECT_THAT(matcher.MatchVariable("foo"), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_Attribute_OneVariable_TwoQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT( + matcher.GetAttributes(), + UnorderedElementsAre( + AttributePattern("foo", {AttributeQualifierPattern::OfString("bar")}), + AttributePattern("foo", + {AttributeQualifierPattern::OfString("baz")}))); + EXPECT_THAT(matcher.MatchVariable("foo"), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("baz")), + AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_AttributePattern_OneVariable_ZeroQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern("foo")), IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern("foo"))); + EXPECT_THAT(matcher.MatchVariable("foo"), AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_AttributePattern_OneVariable_OneQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")}))); + EXPECT_THAT(matcher.MatchVariable("foo"), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, Unknown_AttributePattern_OneVariable_TwoQualifier) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("baz")})), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT( + matcher.GetAttributes(), + UnorderedElementsAre( + AttributePattern("foo", {AttributeQualifierPattern::OfString("bar")}), + AttributePattern("foo", + {AttributeQualifierPattern::OfString("baz")}))); + EXPECT_THAT(matcher.MatchVariable("foo"), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("baz")), + AttributeMatchIs(MatchType::FULL)); +} + +TEST(AttributeMatcher, RemoveAttribute) { + const std::vector unknown_attributes( + {AttributePattern("foo", {AttributeQualifierPattern::OfString("bar")}), + AttributePattern("bar", {AttributeQualifierPattern::OfString("foo")}), + AttributePattern("baz", {AttributeQualifierPattern::OfString("foo")})}); + AttributeMatcher matcher; + std::vector removal_order({0, 1, 2}); + do { + for (const auto& attribute : unknown_attributes) { + EXPECT_THAT(matcher.InsertAttribute(attribute), IsOk()); + } + // 0 + EXPECT_THAT(matcher.RemoveAttribute(unknown_attributes[removal_order[0]]), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(unknown_attributes[removal_order[1]], + unknown_attributes[removal_order[2]])); + // 1 + EXPECT_THAT(matcher.RemoveAttribute(unknown_attributes[removal_order[1]]), + IsOk()); + EXPECT_TRUE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(unknown_attributes[removal_order[2]])); + // 2 + EXPECT_THAT(matcher.RemoveAttribute(unknown_attributes[removal_order[2]]), + IsOk()); + EXPECT_FALSE(matcher.HasAttributes()); + EXPECT_THAT(matcher.GetAttributes(), IsEmpty()); + } while (std::next_permutation(removal_order.begin(), removal_order.end())); +} + +TEST(AttributeMatcher, Wildcards) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::Wildcard(), + AttributeQualifierPattern::Wildcard()})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "foo", {AttributeQualifierPattern::Wildcard()})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute(AttributePattern( + "bar", {AttributeQualifierPattern::OfString("baz"), + AttributeQualifierPattern::Wildcard()})), + IsOk()); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.InsertAttribute( + Attribute("bar", {AttributeQualifier::OfString("baz")})), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfInt(1)), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfUint(1)), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfBool(true)), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT( + matcher.GetAttributes(), + UnorderedElementsAre( + AttributePattern("foo", {AttributeQualifierPattern::Wildcard()}), + AttributePattern("bar", {AttributeQualifierPattern::OfString("baz"), + AttributeQualifierPattern::Wildcard()}))); + EXPECT_THAT(matcher.RemoveAttribute(Attribute("foo")), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute(Attribute("bar")), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute( + Attribute("bar", {AttributeQualifier::OfString("baz")})), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(matcher.RemoveAttribute( + Attribute("bar", {AttributeQualifier::OfString("baz"), + AttributeQualifier::OfString("foo")})), + StatusIs(absl::StatusCode::kNotFound)); +} + +TEST(AttributeMatcher, Upsert) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); + + // Upserting the same attribute does nothing. + EXPECT_THAT(matcher.UpsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); + + // Upserting a more specific attribute does nothing. + EXPECT_THAT(matcher.UpsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz"), + AttributeQualifier::OfString("qux")})), + IsOk()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}))); + + // Upserting a less specific attribute replaces others. + EXPECT_THAT(matcher.UpsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("qux")})), + IsOk()); + EXPECT_THAT( + matcher.GetAttributes(), + UnorderedElementsAre( + AttributePattern("foo", {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz")}), + AttributePattern("foo", + {AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("qux")}))); + EXPECT_THAT(matcher.UpsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + IsOk()); + EXPECT_THAT(matcher.GetAttributes(), + UnorderedElementsAre(AttributePattern( + "foo", {AttributeQualifierPattern::OfString("bar")}))); +} + +TEST(AttributeMatcher, PastFullMatch) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + IsOk()); + EXPECT_DEBUG_DEATH( + static_cast( + matcher.MatchVariable("foo") + .MatchQualifier(AttributeQualifierView::OfString("bar")) + .MatchQualifier(AttributeQualifierView::OfString("baz"))), + _); +} + +} // namespace +} // namespace cel diff --git a/base/attribute_test.cc b/base/attribute_test.cc index d4b36f9f5..0adff75ad 100644 --- a/base/attribute_test.cc +++ b/base/attribute_test.cc @@ -76,5 +76,24 @@ TEST(AttributeQualifierView, String) { EXPECT_THAT(qualifier.AsString(), Optional(absl::string_view("foo"))); } +TEST(Attribute, ToString) { + EXPECT_EQ(Attribute("foo", {AttributeQualifier::OfBool(true), + AttributeQualifier::OfInt(2), + AttributeQualifier::OfUint(3), + AttributeQualifier::OfString("bar")}) + .ToString(), + "foo[true][2][3u][\"bar\"]"); +} + +TEST(AttributePattern, ToString) { + EXPECT_EQ(AttributePattern("foo", {AttributeQualifierPattern::OfBool(true), + AttributeQualifierPattern::OfInt(2), + AttributeQualifierPattern::OfUint(3), + AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::Wildcard()}) + .ToString(), + "foo[true][2][3u][\"bar\"][*]"); +} + } // namespace } // namespace cel From 8f40794278f09867c122333d8638eff7595ba860 Mon Sep 17 00:00:00 2001 From: Justin King Date: Thu, 8 Oct 2026 14:19:06 -0700 Subject: [PATCH 274/350] Inline common `ByteString`, `BytesValue`, and `StringValue` methods PiperOrigin-RevId: 996067905 --- common/internal/BUILD | 1 - common/internal/byte_string.cc | 304 ++++------------------------ common/internal/byte_string.h | 210 ++++++++++++++++--- common/internal/byte_string_test.cc | 16 -- common/values/bytes_value.cc | 69 ------- common/values/bytes_value.h | 28 +-- common/values/string_value.cc | 59 +----- common/values/string_value.h | 51 +++-- 8 files changed, 267 insertions(+), 471 deletions(-) diff --git a/common/internal/BUILD b/common/internal/BUILD index 571f1ef14..99097fb12 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -25,7 +25,6 @@ cc_library( "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/functional:overload", - "@com_google_absl//absl/hash", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", diff --git a/common/internal/byte_string.cc b/common/internal/byte_string.cc index 2b713b7b1..4daaa66a1 100644 --- a/common/internal/byte_string.cc +++ b/common/internal/byte_string.cc @@ -16,7 +16,6 @@ #include #include -#include #include #include #include @@ -24,10 +23,8 @@ #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/functional/overload.h" -#include "absl/hash/hash.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" -#include "absl/strings/match.h" #include "absl/strings/resize_and_overwrite.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -35,69 +32,6 @@ namespace cel::common_internal { -namespace { - -char* CopyCordToArray(const absl::Cord& cord, size_t offset, size_t size, - char* data) { - for (auto chunk : cord.Chunks()) { - if (size == 0) { - break; - } - if (offset > 0) { - size_t min_offset = std::min(chunk.size(), offset); - offset -= min_offset; - if (offset > 0) { - continue; - } - chunk.remove_prefix(min_offset); - } - size_t min_size = std::min(size, chunk.size()); - std::memcpy(data, chunk.data(), min_size); - data += min_size; - size -= min_size; - } - return data; -} - -char* CopyCordToArray(const absl::Cord& cord, char* data) { - return (CopyCordToArray)(cord, 0, cord.size(), data); -} - -void AppendCordToString(const absl::Cord& cord, size_t offset, size_t size, - std::string& data) { - data.reserve(data.size() + size); - for (auto chunk : cord.Chunks()) { - if (size == 0) { - break; - } - if (offset > 0) { - size_t min_offset = std::min(chunk.size(), offset); - offset -= min_offset; - if (offset > 0) { - continue; - } - chunk.remove_prefix(min_offset); - } - size_t min_size = std::min(size, chunk.size()); - data.append(absl::string_view(chunk.data(), min_size)); - size -= min_size; - } -} - -template -T ConsumeAndDestroy(T& object) { - T consumed = std::move(object); - object.~T(); // NOLINT(bugprone-use-after-move) - return consumed; -} - -} // namespace - -ByteString ByteString::From(const char* absl_nullable value, - google::protobuf::Arena* absl_nonnull arena) { - return From(absl::NullSafeStringView(value), arena); -} - ByteString ByteString::From(absl::string_view value, google::protobuf::Arena* absl_nonnull arena) { ABSL_DCHECK(arena != nullptr); @@ -113,14 +47,15 @@ ByteString ByteString::From(absl::string_view value, return result; } -ByteString ByteString::From(const absl::Cord& value, +ByteString ByteString::From(absl::Cord value, google::protobuf::Arena* absl_nonnull arena) { ABSL_DCHECK(arena != nullptr); ByteString result(UninitializedTag{}); if (value.size() <= kSmallByteStringCapacity) { result.SetSmall(arena, value); } else { - result.SetLarge(arena, google::protobuf::Arena::Create(arena, value)); + result.SetLarge(arena, + google::protobuf::Arena::Create(arena, std::move(value))); } return result; } @@ -144,29 +79,6 @@ ByteString ByteString::From(std::string&& value, return result; } -ByteString ByteString::Wrap(absl::string_view value, - google::protobuf::Arena* absl_nullable arena) { - ByteString result(UninitializedTag{}); - result.SetMedium(arena, value); - return result; -} - -ByteString ByteString::Wrap(const absl::Cord* absl_nonnull value, size_t offset, - size_t size, google::protobuf::Arena* absl_nullable arena) { - ByteString result(UninitializedTag{}); - result.SetLarge(arena, value, offset, size); - return result; -} - -ByteString ByteString::WrapUnsafe(absl::string_view value) { - return Wrap(value, static_cast(nullptr)); -} - -ByteString ByteString::WrapUnsafe(const absl::Cord* absl_nonnull value, - size_t offset, size_t size) { - return Wrap(value, offset, size, static_cast(nullptr)); -} - ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, google::protobuf::Arena* absl_nonnull arena) { ABSL_DCHECK(arena != nullptr); @@ -212,39 +124,6 @@ ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, return result; } -google::protobuf::Arena* absl_nullable ByteString::GetArena() const { - switch (GetKind()) { - case ByteStringKind::kSmall: - return GetSmallArena(); - case ByteStringKind::kMedium: - return GetMediumArena(); - case ByteStringKind::kLarge: - return GetLargeArena(); - } -} - -bool ByteString::empty() const { - switch (GetKind()) { - case ByteStringKind::kSmall: - return rep_.small.size == 0; - case ByteStringKind::kMedium: - return rep_.medium.size == 0; - case ByteStringKind::kLarge: - return rep_.large.size == 0; - } -} - -size_t ByteString::size() const { - switch (GetKind()) { - case ByteStringKind::kSmall: - return rep_.small.size; - case ByteStringKind::kMedium: - return rep_.medium.size; - case ByteStringKind::kLarge: - return rep_.large.size; - } -} - absl::optional ByteString::TryFlat() const { switch (GetKind()) { case ByteStringKind::kSmall: @@ -260,63 +139,6 @@ absl::optional ByteString::TryFlat() const { } } -bool ByteString::Equals(absl::string_view rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { return lhs == rhs; }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs == rhs; })); -} - -bool ByteString::Equals(const absl::Cord& rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { return lhs == rhs; }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs == rhs; })); -} - -int ByteString::Compare(absl::string_view rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> int { return lhs.compare(rhs); }, - [&rhs](const absl::Cord& lhs) -> int { return lhs.Compare(rhs); })); -} - -int ByteString::Compare(const absl::Cord& rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> int { return -rhs.Compare(lhs); }, - [&rhs](const absl::Cord& lhs) -> int { return lhs.Compare(rhs); })); -} - -bool ByteString::StartsWith(absl::string_view rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { - return absl::StartsWith(lhs, rhs); - }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs.StartsWith(rhs); })); -} - -bool ByteString::StartsWith(const absl::Cord& rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { - return lhs.size() >= rhs.size() && lhs.substr(0, rhs.size()) == rhs; - }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs.StartsWith(rhs); })); -} - -bool ByteString::EndsWith(absl::string_view rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { - return absl::EndsWith(lhs, rhs); - }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs.EndsWith(rhs); })); -} - -bool ByteString::EndsWith(const absl::Cord& rhs) const { - return Visit(absl::Overload( - [&rhs](absl::string_view lhs) -> bool { - return lhs.size() >= rhs.size() && - lhs.substr(lhs.size() - rhs.size()) == rhs; - }, - [&rhs](const absl::Cord& lhs) -> bool { return lhs.EndsWith(rhs); })); -} - absl::optional ByteString::Find(absl::string_view needle, size_t pos) const { ABSL_DCHECK_LE(pos, size()); @@ -505,8 +327,8 @@ void ByteString::CopyToString(std::string* absl_nonnull out) const { absl::StringResizeAndOverwrite( *out, rep_.large.size, [this](char* buffer, size_t buffer_size) -> size_t { - (CopyCordToArray)(*rep_.large.data, rep_.large.offset, - rep_.large.size, buffer); + CopyCordToArray(*rep_.large.data, rep_.large.offset, + rep_.large.size, buffer); return rep_.large.size; }); break; @@ -523,24 +345,13 @@ void ByteString::AppendToString(std::string* absl_nonnull out) const { out->append(GetMedium()); break; case ByteStringKind::kLarge: - (AppendCordToString)(*rep_.large.data, rep_.large.offset, rep_.large.size, - *out); + AppendCordToString(*rep_.large.data, rep_.large.offset, rep_.large.size, + *out); break; } } -absl::Cord ByteString::ToCord() const& { - switch (GetKind()) { - case ByteStringKind::kSmall: - return absl::Cord(GetSmall()); - case ByteStringKind::kMedium: - return absl::Cord(GetMedium()); - case ByteStringKind::kLarge: - return GetLarge(); - } -} - -absl::Cord ByteString::ToCord() && { +absl::Cord ByteString::ToCord() const { switch (GetKind()) { case ByteStringKind::kSmall: return absl::Cord(GetSmall()); @@ -596,28 +407,14 @@ absl::string_view ByteString::ToStringView( absl::StringResizeAndOverwrite( *scratch, rep_.large.size, [this](char* buffer, size_t buffer_size) -> size_t { - (CopyCordToArray)(*rep_.large.data, rep_.large.offset, - rep_.large.size, buffer); + CopyCordToArray(*rep_.large.data, rep_.large.offset, + rep_.large.size, buffer); return rep_.large.size; }); return absl::string_view(*scratch); } } -absl::string_view ByteString::AsStringView() const { - const ByteStringKind kind = GetKind(); - ABSL_CHECK(kind == ByteStringKind::kSmall || // Crash OK - kind == ByteStringKind::kMedium); - switch (kind) { - case ByteStringKind::kSmall: - return GetSmall(); - case ByteStringKind::kMedium: - return GetMedium(); - case ByteStringKind::kLarge: - ABSL_UNREACHABLE(); - } -} - ByteString ByteString::Clone(google::protobuf::Arena* absl_nonnull arena) const { ABSL_DCHECK(arena != nullptr); switch (GetKind()) { @@ -643,63 +440,38 @@ ByteString ByteString::Clone(google::protobuf::Arena* absl_nonnull arena) const } } -void ByteString::HashValue(absl::HashState state) const { - switch (GetKind()) { - case ByteStringKind::kSmall: - absl::HashState::combine(std::move(state), GetSmall()); - break; - case ByteStringKind::kMedium: - absl::HashState::combine(std::move(state), GetMedium()); - break; - case ByteStringKind::kLarge: - absl::HashState::combine(std::move(state), GetLarge()); +void ByteString::CopyCordToArray(const absl::Cord& cord, size_t offset, + size_t size, char* data) { + for (auto chunk : cord.Chunks()) { + if (size == 0) { break; + } + if (offset > 0) { + size_t min_offset = std::min(chunk.size(), offset); + offset -= min_offset; + if (offset > 0) { + continue; + } + chunk.remove_prefix(min_offset); + } + size_t min_size = std::min(size, chunk.size()); + std::memcpy(data, chunk.data(), min_size); + data += min_size; + size -= min_size; } } -void ByteString::SetSmall(google::protobuf::Arena* absl_nullable arena, - absl::string_view string) { - ABSL_DCHECK_LE(string.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kSmall; - rep_.small.size = string.size(); - rep_.small.arena = arena; - if (!string.empty()) { - std::memcpy(rep_.small.data, string.data(), rep_.small.size); - } -} - -void ByteString::SetSmall(google::protobuf::Arena* absl_nullable arena, - const absl::Cord& cord) { - ABSL_DCHECK_LE(cord.size(), kSmallByteStringCapacity); - rep_.header.kind = ByteStringKind::kSmall; - rep_.small.size = cord.size(); - rep_.small.arena = arena; - (CopyCordToArray)(cord, rep_.small.data); -} - -void ByteString::SetMedium(google::protobuf::Arena* absl_nullable arena, - absl::string_view string) { - rep_.header.kind = ByteStringKind::kMedium; - rep_.medium.size = string.size(); - rep_.medium.data = string.data(); - rep_.medium.arena = arena; -} - -void ByteString::SetLarge(google::protobuf::Arena* absl_nullable arena, - const absl::Cord* absl_nonnull cord, size_t offset, - size_t size) { - ABSL_DCHECK_LE(offset, cord->size()); - ABSL_DCHECK_LE(offset, kLargeByteStringMaxSize); - rep_.header.kind = ByteStringKind::kLarge; - rep_.large.offset = offset; - if (size == static_cast(-1)) { - size = cord->size() - offset; - } - ABSL_DCHECK_LE(size, cord->size() - offset); - ABSL_DCHECK_LE(size, kLargeByteStringMaxSize); - rep_.large.size = size; - rep_.large.data = cord; - rep_.large.arena = arena; +void ByteString::AppendCordToString(const absl::Cord& cord, size_t offset, + size_t size, std::string& data) { + size_t original_size = data.size(); + size_t target_size = original_size + size; + absl::StringResizeAndOverwrite( + data, original_size + size, + [&cord, offset, size, original_size, target_size]( + char* buf, size_t buf_size) -> size_t { + CopyCordToArray(cord, offset, size, buf + original_size); + return target_size; + }); } absl::string_view LegacyByteString(const ByteString& string, bool stable, diff --git a/common/internal/byte_string.h b/common/internal/byte_string.h index de11f8a9e..6027d08ec 100644 --- a/common/internal/byte_string.h +++ b/common/internal/byte_string.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,9 +26,9 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/functional/overload.h" -#include "absl/hash/hash.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" +#include "absl/strings/match.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "google/protobuf/arena.h" @@ -161,20 +162,25 @@ class [[nodiscard]] ByteString final { public: static ByteString From(const char* absl_nullable value, google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + return From(absl::NullSafeStringView(value), arena); + } static ByteString From(absl::string_view value, google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND); - static ByteString From(const absl::Cord& value, - google::protobuf::Arena* absl_nonnull arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + static ByteString From(absl::Cord value, google::protobuf::Arena* absl_nonnull arena + ABSL_ATTRIBUTE_LIFETIME_BOUND); static ByteString From(std::string&& value, google::protobuf::Arena* absl_nonnull arena ABSL_ATTRIBUTE_LIFETIME_BOUND); static ByteString Wrap(absl::string_view value, google::protobuf::Arena* absl_nullable arena - ABSL_ATTRIBUTE_LIFETIME_BOUND); + ABSL_ATTRIBUTE_LIFETIME_BOUND) { + ByteString result(UninitializedTag{}); + result.SetMedium(arena, value); + return result; + } static ByteString Wrap( const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { @@ -183,18 +189,26 @@ class [[nodiscard]] ByteString final { static ByteString Wrap( const absl::Cord* absl_nonnull value ABSL_ATTRIBUTE_LIFETIME_BOUND, size_t offset, size_t size, - google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND); + google::protobuf::Arena* absl_nullable arena ABSL_ATTRIBUTE_LIFETIME_BOUND) { + ByteString result(UninitializedTag{}); + result.SetLarge(arena, value, offset, size); + return result; + } static ByteString Wrap(std::nullptr_t, google::protobuf::Arena*) = delete; static ByteString Wrap(std::nullptr_t, size_t, size_t, google::protobuf::Arena*) = delete; static ByteString Wrap(std::string&& value, google::protobuf::Arena*) = delete; - static ByteString WrapUnsafe(absl::string_view value); + static ByteString WrapUnsafe(absl::string_view value) { + return Wrap(value, static_cast(nullptr)); + } static ByteString WrapUnsafe(const absl::Cord* absl_nonnull value) { return WrapUnsafe(value, 0, value->size()); } static ByteString WrapUnsafe(const absl::Cord* absl_nonnull value, - size_t offset, size_t size); + size_t offset, size_t size) { + return Wrap(value, offset, size, static_cast(nullptr)); + } static ByteString WrapUnsafe(std::nullptr_t) = delete; static ByteString WrapUnsafe(std::nullptr_t, size_t, size_t) = delete; @@ -208,9 +222,27 @@ class [[nodiscard]] ByteString final { ByteString& operator=(const ByteString&) = default; ByteString& operator=(ByteString&&) = default; - bool empty() const; + bool empty() const { + switch (GetKind()) { + case ByteStringKind::kSmall: + return rep_.small.size == 0; + case ByteStringKind::kMedium: + return rep_.medium.size == 0; + case ByteStringKind::kLarge: + return rep_.large.size == 0; + } + } - size_t size() const; + size_t size() const { + switch (GetKind()) { + case ByteStringKind::kSmall: + return rep_.small.size; + case ByteStringKind::kMedium: + return rep_.medium.size; + case ByteStringKind::kLarge: + return rep_.large.size; + } + } size_t max_size() const { return kByteStringViewMaxSize; } @@ -261,9 +293,7 @@ class [[nodiscard]] ByteString final { void AppendToString(std::string* absl_nonnull out) const; - absl::Cord ToCord() const&; - - absl::Cord ToCord() &&; + absl::Cord ToCord() const; void CopyToCord(absl::Cord* absl_nonnull out) const; @@ -273,24 +303,25 @@ class [[nodiscard]] ByteString final { std::string* absl_nonnull scratch ABSL_ATTRIBUTE_LIFETIME_BOUND) const ABSL_ATTRIBUTE_LIFETIME_BOUND; - absl::string_view AsStringView() const ABSL_ATTRIBUTE_LIFETIME_BOUND; - - google::protobuf::Arena* absl_nullable GetArena() const; + google::protobuf::Arena* absl_nullable GetArena() const { + switch (GetKind()) { + case ByteStringKind::kSmall: + return GetSmallArena(); + case ByteStringKind::kMedium: + return GetMediumArena(); + case ByteStringKind::kLarge: + return GetLargeArena(); + } + } ByteString Clone(google::protobuf::Arena* absl_nonnull arena) const; - void HashValue(absl::HashState state) const; - template decltype(auto) Visit(Visitor&& visitor) const { - switch (GetKind()) { - case ByteStringKind::kSmall: - return std::forward(visitor)(GetSmall()); - case ByteStringKind::kMedium: - return std::forward(visitor)(GetMedium()); - case ByteStringKind::kLarge: - return std::forward(visitor)(GetLarge()); + if (auto string_view = AsStringView(); string_view) { + return std::forward(visitor)(*string_view); } + return std::forward(visitor)(GetLarge()); } friend void swap(ByteString& lhs, ByteString& rhs) noexcept { @@ -300,8 +331,10 @@ class [[nodiscard]] ByteString final { template friend H AbslHashValue(H state, const ByteString& byte_string) { - byte_string.HashValue(absl::HashState::Create(&state)); - return state; + if (auto string_view = byte_string.AsStringView(); string_view) { + return H::combine(std::move(state), *string_view); + } + return H::combine(std::move(state), byte_string.GetLarge()); } private: @@ -386,11 +419,30 @@ class [[nodiscard]] ByteString final { rep_.small.arena = arena; } - void SetSmall(google::protobuf::Arena* absl_nullable arena, absl::string_view string); + void SetSmall(google::protobuf::Arena* absl_nullable arena, absl::string_view string) { + ABSL_DCHECK_LE(string.size(), kSmallByteStringCapacity); + rep_.header.kind = ByteStringKind::kSmall; + rep_.small.size = string.size(); + rep_.small.arena = arena; + if (!string.empty()) { + std::memcpy(rep_.small.data, string.data(), rep_.small.size); + } + } - void SetSmall(google::protobuf::Arena* absl_nullable arena, const absl::Cord& cord); + void SetSmall(google::protobuf::Arena* absl_nullable arena, const absl::Cord& cord) { + ABSL_DCHECK_LE(cord.size(), kSmallByteStringCapacity); + rep_.header.kind = ByteStringKind::kSmall; + rep_.small.size = cord.size(); + rep_.small.arena = arena; + CopyCordToArray(cord, rep_.small.data); + } - void SetMedium(google::protobuf::Arena* absl_nullable arena, absl::string_view string); + void SetMedium(google::protobuf::Arena* absl_nullable arena, absl::string_view string) { + rep_.header.kind = ByteStringKind::kMedium; + rep_.medium.size = string.size(); + rep_.medium.data = string.data(); + rep_.medium.arena = arena; + } void SetMedium(google::protobuf::Arena* absl_nullable arena, const std::string* absl_nonnull string) { @@ -399,31 +451,125 @@ class [[nodiscard]] ByteString final { void SetLarge(google::protobuf::Arena* absl_nullable arena, const absl::Cord* absl_nonnull cord, size_t offset = 0, - size_t size = static_cast(-1)); + size_t size = static_cast(-1)) { + ABSL_DCHECK_LE(offset, cord->size()); + ABSL_DCHECK_LE(offset, kLargeByteStringMaxSize); + rep_.header.kind = ByteStringKind::kLarge; + rep_.large.offset = offset; + if (size == static_cast(-1)) { + size = cord->size() - offset; + } + ABSL_DCHECK_LE(size, cord->size() - offset); + ABSL_DCHECK_LE(size, kLargeByteStringMaxSize); + rep_.large.size = size; + rep_.large.data = cord; + rep_.large.arena = arena; + } + + static void CopyCordToArray(const absl::Cord& cord, char* data) { + for (absl::string_view chunk : cord.Chunks()) { + std::memcpy(data, chunk.data(), chunk.size()); + data += chunk.size(); + } + } + + static void CopyCordToArray(const absl::Cord& cord, size_t offset, + size_t size, char* data); + + static void AppendCordToString(const absl::Cord& cord, size_t offset, + size_t size, std::string& data); void CopyToArray(char* absl_nonnull out) const; + std::optional AsStringView() const { + switch (GetKind()) { + case ByteStringKind::kSmall: + return GetSmall(); + case ByteStringKind::kMedium: + return GetMedium(); + case ByteStringKind::kLarge: + return std::nullopt; + } + } + ByteStringRep rep_; }; +inline bool ByteString::Equals(absl::string_view rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { return lhs == rhs; }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs == rhs; })); +} + +inline bool ByteString::Equals(const absl::Cord& rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { return lhs == rhs; }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs == rhs; })); +} + inline bool ByteString::Equals(const ByteString& rhs) const { return rhs.Visit(absl::Overload( [this](absl::string_view rhs) -> bool { return Equals(rhs); }, [this](const absl::Cord& rhs) -> bool { return Equals(rhs); })); } +inline int ByteString::Compare(absl::string_view rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> int { return lhs.compare(rhs); }, + [&rhs](const absl::Cord& lhs) -> int { return lhs.Compare(rhs); })); +} + +inline int ByteString::Compare(const absl::Cord& rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> int { return -rhs.Compare(lhs); }, + [&rhs](const absl::Cord& lhs) -> int { return lhs.Compare(rhs); })); +} + inline int ByteString::Compare(const ByteString& rhs) const { return rhs.Visit(absl::Overload( [this](absl::string_view rhs) -> int { return Compare(rhs); }, [this](const absl::Cord& rhs) -> int { return Compare(rhs); })); } +inline bool ByteString::StartsWith(absl::string_view rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { + return absl::StartsWith(lhs, rhs); + }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs.StartsWith(rhs); })); +} + +inline bool ByteString::StartsWith(const absl::Cord& rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { + return lhs.size() >= rhs.size() && lhs.substr(0, rhs.size()) == rhs; + }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs.StartsWith(rhs); })); +} + inline bool ByteString::StartsWith(const ByteString& rhs) const { return rhs.Visit(absl::Overload( [this](absl::string_view rhs) -> bool { return StartsWith(rhs); }, [this](const absl::Cord& rhs) -> bool { return StartsWith(rhs); })); } +inline bool ByteString::EndsWith(absl::string_view rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { + return absl::EndsWith(lhs, rhs); + }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs.EndsWith(rhs); })); +} + +inline bool ByteString::EndsWith(const absl::Cord& rhs) const { + return Visit(absl::Overload( + [&rhs](absl::string_view lhs) -> bool { + return lhs.size() >= rhs.size() && + lhs.substr(lhs.size() - rhs.size()) == rhs; + }, + [&rhs](const absl::Cord& lhs) -> bool { return lhs.EndsWith(rhs); })); +} + inline bool ByteString::EndsWith(const ByteString& rhs) const { return rhs.Visit(absl::Overload( [this](absl::string_view rhs) -> bool { return EndsWith(rhs); }, diff --git a/common/internal/byte_string_test.cc b/common/internal/byte_string_test.cc index b2353ac2d..121158ada 100644 --- a/common/internal/byte_string_test.cc +++ b/common/internal/byte_string_test.cc @@ -43,7 +43,6 @@ using ::testing::IsEmpty; using ::testing::Not; using ::testing::Optional; using ::testing::SizeIs; -using ::testing::TestWithParam; TEST(ByteStringKind, Ostream) { { @@ -496,21 +495,6 @@ TEST_F(ByteStringTest, ToStringViewLarge) { EXPECT_EQ(byte_string.ToStringView(&scratch), GetMediumOrLargeCord()); } -TEST_F(ByteStringTest, AsStringViewSmall) { - ByteString byte_string = ByteString::WrapUnsafe(GetSmallStringView()); - EXPECT_EQ(byte_string.AsStringView(), GetSmallStringView()); -} - -TEST_F(ByteStringTest, AsStringViewMedium) { - ByteString byte_string = ByteString::WrapUnsafe(GetMediumStringView()); - EXPECT_EQ(byte_string.AsStringView(), GetMediumStringView()); -} - -TEST_F(ByteStringTest, AsStringViewLarge) { - ByteString byte_string = ByteString::WrapUnsafe(&GetMediumOrLargeCord()); - EXPECT_DEATH(byte_string.AsStringView(), _); -} - TEST_F(ByteStringTest, CopyToStringSmall) { std::string out; diff --git a/common/values/bytes_value.cc b/common/values/bytes_value.cc index c9fc32ac2..5c57bd740 100644 --- a/common/values/bytes_value.cc +++ b/common/values/bytes_value.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include "google/protobuf/wrappers.pb.h" @@ -123,72 +122,4 @@ absl::Status BytesValue::Equal( return absl::OkStatus(); } -BytesValue BytesValue::Clone(google::protobuf::Arena* absl_nonnull arena) const { - return BytesValue(value_.Clone(arena)); -} - -size_t BytesValue::Size() const { - return NativeValue( - [](const auto& alternative) -> size_t { return alternative.size(); }); -} - -bool BytesValue::IsEmpty() const { - return NativeValue( - [](const auto& alternative) -> bool { return alternative.empty(); }); -} - -bool BytesValue::Equals(absl::string_view bytes) const { - return NativeValue([bytes](const auto& alternative) -> bool { - return alternative == bytes; - }); -} - -bool BytesValue::Equals(const absl::Cord& bytes) const { - return NativeValue([&bytes](const auto& alternative) -> bool { - return alternative == bytes; - }); -} - -bool BytesValue::Equals(const BytesValue& bytes) const { - return bytes.NativeValue( - [this](const auto& alternative) -> bool { return Equals(alternative); }); -} - -namespace { - -int CompareImpl(absl::string_view lhs, absl::string_view rhs) { - return lhs.compare(rhs); -} - -int CompareImpl(absl::string_view lhs, const absl::Cord& rhs) { - return -rhs.Compare(lhs); -} - -int CompareImpl(const absl::Cord& lhs, absl::string_view rhs) { - return lhs.Compare(rhs); -} - -int CompareImpl(const absl::Cord& lhs, const absl::Cord& rhs) { - return lhs.Compare(rhs); -} - -} // namespace - -int BytesValue::Compare(absl::string_view bytes) const { - return NativeValue([bytes](const auto& alternative) -> int { - return CompareImpl(alternative, bytes); - }); -} - -int BytesValue::Compare(const absl::Cord& bytes) const { - return NativeValue([&bytes](const auto& alternative) -> int { - return CompareImpl(alternative, bytes); - }); -} - -int BytesValue::Compare(const BytesValue& bytes) const { - return bytes.NativeValue( - [this](const auto& alternative) -> int { return Compare(alternative); }); -} - } // namespace cel diff --git a/common/values/bytes_value.h b/common/values/bytes_value.h index 461e5d8c6..42be2120e 100644 --- a/common/values/bytes_value.h +++ b/common/values/bytes_value.h @@ -138,11 +138,11 @@ class BytesValue final : private common_internal::ValueMixin { Value* absl_nonnull result) const; using ValueMixin::Equal; - bool IsZeroValue() const { - return NativeValue([](const auto& value) -> bool { return value.empty(); }); - } + bool IsZeroValue() const { return value_.empty(); } - BytesValue Clone(google::protobuf::Arena* absl_nonnull arena) const; + BytesValue Clone(google::protobuf::Arena* absl_nonnull arena) const { + return BytesValue(value_.Clone(arena)); + } ABSL_DEPRECATED("Use ToString()") std::string NativeString() const { return value_.ToString(); } @@ -172,17 +172,21 @@ class BytesValue final : private common_internal::ValueMixin { swap(lhs.value_, rhs.value_); } - size_t Size() const; + size_t Size() const { return value_.size(); } - bool IsEmpty() const; + bool IsEmpty() const { return value_.empty(); } - bool Equals(absl::string_view bytes) const; - bool Equals(const absl::Cord& bytes) const; - bool Equals(const BytesValue& bytes) const; + bool Equals(absl::string_view bytes) const { return value_.Equals(bytes); } + bool Equals(const absl::Cord& bytes) const { return value_.Equals(bytes); } + bool Equals(const BytesValue& bytes) const { + return value_.Equals(bytes.value_); + } - int Compare(absl::string_view bytes) const; - int Compare(const absl::Cord& bytes) const; - int Compare(const BytesValue& bytes) const; + int Compare(absl::string_view bytes) const { return value_.Compare(bytes); } + int Compare(const absl::Cord& bytes) const { return value_.Compare(bytes); } + int Compare(const BytesValue& bytes) const { + return value_.Compare(bytes.value_); + } absl::optional TryFlat() const ABSL_ATTRIBUTE_LIFETIME_BOUND { diff --git a/common/values/string_value.cc b/common/values/string_value.cc index 053a456bd..6aa7ac543 100644 --- a/common/values/string_value.cc +++ b/common/values/string_value.cc @@ -138,68 +138,11 @@ absl::Status StringValue::Equal( } size_t StringValue::Size() const { - return NativeValue([](const auto& alternative) -> size_t { + return value_.Visit([](const auto& alternative) -> size_t { return internal::Utf8CodePointCount(alternative); }); } -bool StringValue::IsEmpty() const { - return NativeValue( - [](const auto& alternative) -> bool { return alternative.empty(); }); -} - -bool StringValue::Equals(absl::string_view string) const { - return value_.Equals(string); -} - -bool StringValue::Equals(const absl::Cord& string) const { - return value_.Equals(string); -} - -bool StringValue::Equals(const StringValue& string) const { - return value_.Equals(string.value_); -} - -StringValue StringValue::Clone(google::protobuf::Arena* absl_nonnull arena) const { - return StringValue(value_.Clone(arena)); -} - -int StringValue::Compare(absl::string_view string) const { - return value_.Compare(string); -} - -int StringValue::Compare(const absl::Cord& string) const { - return value_.Compare(string); -} - -int StringValue::Compare(const StringValue& string) const { - return value_.Compare(string.value_); -} - -bool StringValue::StartsWith(absl::string_view string) const { - return value_.StartsWith(string); -} - -bool StringValue::StartsWith(const absl::Cord& string) const { - return value_.StartsWith(string); -} - -bool StringValue::StartsWith(const StringValue& string) const { - return value_.StartsWith(string.value_); -} - -bool StringValue::EndsWith(absl::string_view string) const { - return value_.EndsWith(string); -} - -bool StringValue::EndsWith(const absl::Cord& string) const { - return value_.EndsWith(string); -} - -bool StringValue::EndsWith(const StringValue& string) const { - return value_.EndsWith(string.value_); -} - bool StringValue::Contains(absl::string_view string) const { return value_.Visit(absl::Overload( [&](absl::string_view lhs) -> bool { diff --git a/common/values/string_value.h b/common/values/string_value.h index 98ca43234..0959cc556 100644 --- a/common/values/string_value.h +++ b/common/values/string_value.h @@ -27,6 +27,7 @@ #include "absl/base/attributes.h" #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" @@ -174,12 +175,12 @@ class StringValue final : private common_internal::ValueMixin { Value* absl_nonnull result) const; using ValueMixin::Equal; - StringValue Clone(google::protobuf::Arena* absl_nonnull arena) const; - - bool IsZeroValue() const { - return NativeValue([](const auto& value) -> bool { return value.empty(); }); + StringValue Clone(google::protobuf::Arena* absl_nonnull arena) const { + return StringValue(value_.Clone(arena)); } + bool IsZeroValue() const { return value_.empty(); } + ABSL_DEPRECATED("Use ToString()") std::string NativeString() const { return value_.ToString(); } @@ -210,23 +211,39 @@ class StringValue final : private common_internal::ValueMixin { size_t Size() const; - bool IsEmpty() const; + bool IsEmpty() const { return value_.empty(); } - bool Equals(absl::string_view string) const; - bool Equals(const absl::Cord& string) const; - bool Equals(const StringValue& string) const; + bool Equals(absl::string_view string) const { return value_.Equals(string); } + bool Equals(const absl::Cord& string) const { return value_.Equals(string); } + bool Equals(const StringValue& string) const { + return value_.Equals(string.value_); + } - int Compare(absl::string_view string) const; - int Compare(const absl::Cord& string) const; - int Compare(const StringValue& string) const; + int Compare(absl::string_view string) const { return value_.Compare(string); } + int Compare(const absl::Cord& string) const { return value_.Compare(string); } + int Compare(const StringValue& string) const { + return value_.Compare(string.value_); + } - bool StartsWith(absl::string_view string) const; - bool StartsWith(const absl::Cord& string) const; - bool StartsWith(const StringValue& string) const; + bool StartsWith(absl::string_view string) const { + return value_.StartsWith(string); + } + bool StartsWith(const absl::Cord& string) const { + return value_.StartsWith(string); + } + bool StartsWith(const StringValue& string) const { + return value_.StartsWith(string.value_); + } - bool EndsWith(absl::string_view string) const; - bool EndsWith(const absl::Cord& string) const; - bool EndsWith(const StringValue& string) const; + bool EndsWith(absl::string_view string) const { + return value_.EndsWith(string); + } + bool EndsWith(const absl::Cord& string) const { + return value_.EndsWith(string); + } + bool EndsWith(const StringValue& string) const { + return value_.EndsWith(string.value_); + } bool Contains(absl::string_view string) const; bool Contains(const absl::Cord& string) const; From 6d152821a5667f7112ce8971dbfce58d1119efe6 Mon Sep 17 00:00:00 2001 From: Justin King Date: Thu, 8 Oct 2026 14:30:33 -0700 Subject: [PATCH 275/350] Change Activation attribute pattern mutators to return `absl::Status` PiperOrigin-RevId: 996074623 --- eval/compiler/BUILD | 1 + .../flat_expr_builder_comprehensions_test.cc | 9 +- ...ilder_short_circuiting_conformance_test.cc | 42 +++++--- eval/compiler/flat_expr_builder_test.cc | 4 +- eval/eval/BUILD | 7 +- eval/eval/comprehension_step_test.cc | 16 +-- eval/eval/container_access_step_test.cc | 3 +- eval/eval/create_list_step_test.cc | 12 ++- eval/eval/equality_steps_test.cc | 20 ++-- eval/eval/function_step_test.cc | 2 +- eval/eval/ident_step_test.cc | 16 +-- eval/eval/select_step_test.cc | 35 +++--- eval/public/BUILD | 1 + eval/public/activation.h | 15 ++- eval/public/activation_test.cc | 4 +- eval/tests/unknowns_end_to_end_test.cc | 94 +++++++++------- extensions/bindings_ext_test.cc | 61 ++++++----- extensions/select_optimization_test.cc | 31 +++--- runtime/BUILD | 1 + runtime/activation.h | 13 ++- runtime/activation_test.cc | 101 ++++++++++-------- 21 files changed, 296 insertions(+), 192 deletions(-) diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index e71c1cdf2..89677776c 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -511,6 +511,7 @@ cc_test( "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", ], diff --git a/eval/compiler/flat_expr_builder_comprehensions_test.cc b/eval/compiler/flat_expr_builder_comprehensions_test.cc index 9d46d8dd8..ecea6058c 100644 --- a/eval/compiler/flat_expr_builder_comprehensions_test.cc +++ b/eval/compiler/flat_expr_builder_comprehensions_test.cc @@ -43,6 +43,7 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::CheckedExpr; @@ -152,9 +153,11 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ListCompWithUnknowns) { &parsed_expr.source_info())); Activation activation; - activation.set_unknown_attribute_patterns({CelAttributePattern{ - "items", - {CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1))}}}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern{ + "items", + {CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1))}}}), + IsOk()); ContainerBackedListImpl list_impl = ContainerBackedListImpl({ CelValue::CreateInt64(1), // element items[1] is marked unknown, so the computation should produce diff --git a/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc b/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc index c00b66824..641ca9cd1 100644 --- a/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc +++ b/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc @@ -6,6 +6,7 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" +#include "absl/status/status_matchers.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "eval/compiler/cel_expression_builder_flat_impl.h" @@ -26,6 +27,7 @@ namespace google::api::expr::runtime { namespace { +using ::absl_testing::IsOk; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::expr::Expr; using ::google::api::expr::parser::Parse; @@ -187,7 +189,9 @@ TEST_P(ShortCircuitingTest, UnknownAnd) { auto builder = GetBuilder(/* enable_unknowns=*/true); absl::Status error = absl::InternalError("error"); - activation.set_unknown_attribute_patterns({CelAttributePattern("var1", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("var1", {})}), + IsOk()); activation.InsertValue("var2", CelValue::CreateError(&error)); activation.InsertValue("var3", CelValue::CreateBool(false)); @@ -217,7 +221,9 @@ TEST_P(ShortCircuitingTest, UnknownOr) { auto builder = GetBuilder(/* enable_unknowns=*/true); absl::Status error = absl::InternalError("error"); - activation.set_unknown_attribute_patterns({CelAttributePattern("var1", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("var1", {})}), + IsOk()); activation.InsertValue("var2", CelValue::CreateError(&error)); activation.InsertValue("var3", CelValue::CreateBool(true)); @@ -307,7 +313,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownCondHandling) { activation.InsertValue("arg1", CelValue::CreateError(&error)); activation.InsertValue("arg2", CelValue::CreateInt64(-1)); - activation.set_unknown_attribute_patterns({CelAttributePattern("cond", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("cond", {})}), + IsOk()); CelValue result; ASSERT_NO_FATAL_FAILURE( @@ -319,9 +327,11 @@ TEST_P(ShortCircuitingTest, TernaryUnknownCondHandling) { EXPECT_THAT(attrs.begin()->variable_name(), Eq("cond")); // Unknown branches are discarded if condition is unknown - activation.set_unknown_attribute_patterns({CelAttributePattern("cond", {}), - CelAttributePattern("arg1", {}), - CelAttributePattern("arg2", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("cond", {}), + CelAttributePattern("arg1", {}), + CelAttributePattern("arg2", {})}), + IsOk()); ASSERT_NO_FATAL_FAILURE( BuildAndEval(builder.get(), expr, activation, &arena, &result)); @@ -344,7 +354,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownArgsHandling) { activation.InsertValue("arg2", CelValue::CreateInt64(-1)); // Unknown arg is discarded if condition chooses other branch. - activation.set_unknown_attribute_patterns({CelAttributePattern("arg1", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("arg1", {})}), + IsOk()); CelValue result; @@ -354,8 +366,10 @@ TEST_P(ShortCircuitingTest, TernaryUnknownArgsHandling) { EXPECT_EQ(result.Int64OrDie(), -1); // Branches won't merge if both are unknown. - activation.set_unknown_attribute_patterns( - {CelAttributePattern("arg1", {}), CelAttributePattern("arg2", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns( + {CelAttributePattern("arg1", {}), CelAttributePattern("arg2", {})}), + IsOk()); ASSERT_NO_FATAL_FAILURE( BuildAndEval(builder.get(), expr, activation, &arena, &result)); @@ -378,8 +392,10 @@ TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { activation.InsertValue("arg2", CelValue::CreateInt64(-1)); // Error cond discards args - activation.set_unknown_attribute_patterns( - {CelAttributePattern("arg1", {}), CelAttributePattern("arg2", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns( + {CelAttributePattern("arg1", {}), CelAttributePattern("arg2", {})}), + IsOk()); CelValue result; @@ -389,7 +405,9 @@ TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { EXPECT_EQ(*result.ErrorOrDie(), error); // Error arg discarded if condition unknown - activation.set_unknown_attribute_patterns({CelAttributePattern("cond", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("cond", {})}), + IsOk()); ASSERT_TRUE(activation.RemoveValueEntry("arg1")); activation.InsertValue("arg1", CelValue::CreateError(&error)); diff --git a/eval/compiler/flat_expr_builder_test.cc b/eval/compiler/flat_expr_builder_test.cc index 105060282..f7cde337b 100644 --- a/eval/compiler/flat_expr_builder_test.cc +++ b/eval/compiler/flat_expr_builder_test.cc @@ -2954,7 +2954,9 @@ TEST_P(FlatExprBuilderVariadicLogicalTest, Evaluate) { insert_value("c", test_case.c_val); if (!unknown_patterns.empty()) { - activation.set_unknown_attribute_patterns(std::move(unknown_patterns)); + ASSERT_THAT( + activation.SetUnknownAttributePatterns(std::move(unknown_patterns)), + IsOk()); } ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr->Evaluate(activation, &arena)); diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 6338b9461..ded2492ca 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -418,7 +418,6 @@ cc_test( deps = [ ":attribute_trail", ":direct_expression_step", - ":equality_steps", ":evaluator_core", "//base:attributes", "//common:value", @@ -452,7 +451,6 @@ cc_test( ":direct_expression_step", ":evaluator_core", ":expression_step_base", - ":ident_step", "//base:data", "//common:expr", "//common:value", @@ -469,7 +467,6 @@ cc_test( "//runtime:runtime_options", "//runtime/internal:runtime_env_testing", "//runtime/internal:runtime_type_provider", - "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -519,7 +516,6 @@ cc_test( ":container_access_step", ":direct_expression_step", ":evaluator_core", - ":ident_step", "//base:builtins", "//base:data", "//common:ast", @@ -540,6 +536,7 @@ cc_test( "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -608,8 +605,6 @@ cc_test( ":const_value_step", ":direct_expression_step", ":evaluator_core", - ":function_step", - ":ident_step", "//base:builtins", "//base:data", "//common:constant", diff --git a/eval/eval/comprehension_step_test.cc b/eval/eval/comprehension_step_test.cc index 7725c4a73..09c4eb228 100644 --- a/eval/eval/comprehension_step_test.cc +++ b/eval/eval/comprehension_step_test.cc @@ -113,11 +113,13 @@ TEST_F(ListKeysStepTest, MapPartiallyUnknown) { (*value.mutable_fields())["key3"].set_number_value(3.0); activation.InsertValue("var", CelProtoWrapper::CreateMessage(&value, &arena)); - activation.set_unknown_attribute_patterns({CelAttributePattern( - "var", - {CreateCelAttributeQualifierPattern(CelValue::CreateStringView("key2")), - CreateCelAttributeQualifierPattern(CelValue::CreateStringView("foo")), - CelAttributeQualifierPattern::CreateWildcard()})}); + ASSERT_THAT(activation.SetUnknownAttributePatterns({CelAttributePattern( + "var", {CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("key2")), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("foo")), + CelAttributeQualifierPattern::CreateWildcard()})}), + IsOk()); auto eval_result = expression->Evaluate(activation, &arena); @@ -171,7 +173,9 @@ TEST_F(ListKeysStepTest, UnknownSetPassedThrough) { Activation activation; Arena arena; - activation.set_unknown_attribute_patterns({CelAttributePattern("var", {})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern("var", {})}), + IsOk()); auto eval_result = expression->Evaluate(activation, &arena); diff --git a/eval/eval/container_access_step_test.cc b/eval/eval/container_access_step_test.cc index 5001c93ef..612732b43 100644 --- a/eval/eval/container_access_step_test.cc +++ b/eval/eval/container_access_step_test.cc @@ -9,6 +9,7 @@ #include "cel/expr/syntax.pb.h" #include "google/protobuf/struct.pb.h" #include "absl/base/nullability.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "base/builtins.h" #include "base/type_provider.h" @@ -101,7 +102,7 @@ CelValue EvaluateAttributeHelper( activation.InsertValue("container", container); activation.InsertValue("key", key); - activation.set_unknown_attribute_patterns(patterns); + ABSL_CHECK_OK(activation.SetUnknownAttributePatterns(patterns)); // Crash OK auto result = cel_expr.Evaluate(activation, arena); return *result; } diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index 119504a14..96da03ed2 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -423,8 +423,10 @@ TEST(CreateDirectListStep, MissingAttribute) { cel::RuntimeOptions options; options.enable_missing_attribute_errors = true; - activation.SetMissingPatterns({cel::AttributePattern( - "var1", {cel::AttributeQualifierPattern::OfString("field1")})}); + ASSERT_THAT( + activation.SetMissingPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::OfString("field1")})}), + IsOk()); ExecutionFrameBase frame(activation, options, type_provider, cel::internal::GetTestingDescriptorPool(), @@ -519,8 +521,10 @@ TEST(CreateDirectListStep, PartialUnknown) { cel::Activation activation; cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - activation.SetUnknownPatterns({cel::AttributePattern( - "var1", {cel::AttributeQualifierPattern::OfString("field1")})}); + ASSERT_THAT( + activation.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::OfString("field1")})}), + IsOk()); ExecutionFrameBase frame(activation, options, type_provider, cel::internal::GetTestingDescriptorPool(), diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index f479d5d3b..660bebeb8 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -94,8 +94,9 @@ TEST(RecursiveTest, PartialAttrUnknown) { std::make_unique(IntValue(1), cel::Attribute("foo")), std::make_unique(IntValue(2)), false, -1); - activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); + ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( + "foo", {cel::AttributeQualifierPattern::OfString("bar")})}), + IsOk()); ExecutionFrameBase frame(activation, opts, type_provider, cel::internal::GetTestingDescriptorPool(), @@ -120,8 +121,9 @@ TEST(RecursiveTest, PartialAttrUnknownDisabled) { std::make_unique(IntValue(1), cel::Attribute("foo")), std::make_unique(IntValue(2)), false, -1); - activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); + ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( + "foo", {cel::AttributeQualifierPattern::OfString("bar")})}), + IsOk()); ExecutionFrameBase frame(activation, opts, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); @@ -154,8 +156,9 @@ TEST(IterativeTest, PartialAttrUnknown) { std::make_unique(IntValue(2)))); steps.push_back(ExpressionStep::MakeFastEqualStep()); - activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); + ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( + "foo", {cel::AttributeQualifierPattern::OfString("bar")})}), + IsOk()); ExecutionFrame frame(steps, activation, opts, state); @@ -185,8 +188,9 @@ TEST(IterativeTest, PartialAttrUnknownDisabled) { std::make_unique(IntValue(2)))); steps.push_back(ExpressionStep::MakeFastEqualStep()); - activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}); + ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( + "foo", {cel::AttributeQualifierPattern::OfString("bar")})}), + IsOk()); ExecutionFrame frame(steps, activation, opts, state); ASSERT_OK_AND_ASSIGN(Value result, frame.Evaluate()); diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index 2946a087a..c6de63d42 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -693,7 +693,7 @@ TEST_P(FunctionStepTestUnknowns, PartialUnknownHandlingTest) { // Set attribute pattern that marks attribute "param[true]" as unknown. // It should result in "param" being handled as partially unknown, which is // is handled as fully unknown when used as function input argument. - activation.set_unknown_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetUnknownAttributePatterns({pattern}), IsOk()); ASSERT_OK_AND_ASSIGN(CelValue value, impl->Evaluate(activation, &arena)); ASSERT_TRUE(value.IsUnknownSet()); diff --git a/eval/eval/ident_step_test.cc b/eval/eval/ident_step_test.cc index f27eaf477..91b2cb986 100644 --- a/eval/eval/ident_step_test.cc +++ b/eval/eval/ident_step_test.cc @@ -114,7 +114,7 @@ TEST(IdentStepTest, DisableMissingAttributeErrorsOK) { EXPECT_THAT(result.StringOrDie().value(), Eq("test")); const CelAttributePattern pattern("name0", {}); - activation.set_missing_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({pattern}), IsOk()); status0 = impl.Evaluate(activation, &arena); ASSERT_THAT(status0, IsOk()); @@ -151,7 +151,7 @@ TEST(IdentStepTest, TestIdentStepMissingAttributeErrors) { EXPECT_THAT(result.StringOrDie().value(), Eq("test")); CelAttributePattern pattern("name0", {}); - activation.set_missing_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({pattern}), IsOk()); status0 = impl.Evaluate(activation, &arena); ASSERT_THAT(status0, IsOk()); @@ -182,7 +182,7 @@ TEST(IdentStepTest, TestIdentStepUnknownAttribute) { std::vector unknown_patterns; unknown_patterns.push_back(CelAttributePattern("name_bad", {})); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), IsOk()); auto status0 = impl.Evaluate(activation, &arena); ASSERT_THAT(status0, IsOk()); @@ -193,7 +193,7 @@ TEST(IdentStepTest, TestIdentStepUnknownAttribute) { unknown_patterns.push_back(CelAttributePattern("name0", {})); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), IsOk()); status0 = impl.Evaluate(activation, &arena); ASSERT_THAT(status0, IsOk()); @@ -234,7 +234,9 @@ TEST(DirectIdentStepTest, UnknownAttribute) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; activation.InsertOrAssignValue("var1", IntValue(42)); - activation.SetUnknownPatterns({CreateCelAttributePattern("var1", {})}); + ASSERT_THAT( + activation.SetUnknownPatterns({CreateCelAttributePattern("var1", {})}), + IsOk()); ExecutionFrameBase frame(activation, options, type_provider, cel::internal::GetTestingDescriptorPool(), @@ -259,7 +261,9 @@ TEST(DirectIdentStepTest, MissingAttribute) { options.enable_missing_attribute_errors = true; activation.InsertOrAssignValue("var1", IntValue(42)); - activation.SetMissingPatterns({CreateCelAttributePattern("var1", {})}); + ASSERT_THAT( + activation.SetMissingPatterns({CreateCelAttributePattern("var1", {})}), + IsOk()); ExecutionFrameBase frame(activation, options, type_provider, cel::internal::GetTestingDescriptorPool(), diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index a1e5b4a09..1fca8fa57 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -807,7 +807,7 @@ TEST_F(SelectStepTest, DisableMissingAttributeOK) { EXPECT_EQ(result.BoolOrDie(), true); CelAttributePattern pattern("message", {}); - activation.set_missing_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({pattern}), IsOk()); ASSERT_OK_AND_ASSIGN(result, cel_expr.Evaluate(activation, &arena_)); EXPECT_EQ(result.BoolOrDie(), true); @@ -853,7 +853,7 @@ TEST_F(SelectStepTest, UnrecoverableUnknownValueProducesError) { CelAttributePattern pattern("message", {CreateCelAttributeQualifierPattern( CelValue::CreateStringView("bool_value"))}); - activation.set_missing_attribute_patterns({pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({pattern}), IsOk()); ASSERT_OK_AND_ASSIGN(result, cel_expr.Evaluate(activation, &arena_)); EXPECT_THAT(*result.ErrorOrDie(), @@ -897,7 +897,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -914,7 +915,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -929,7 +931,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -943,7 +946,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -958,7 +962,8 @@ TEST_F(SelectStepTest, UnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), + IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); @@ -1075,7 +1080,7 @@ TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { Activation activation; activation.InsertValue("message", CelProtoWrapper::CreateMessage(&message, &arena_)); - activation.set_unknown_attribute_patterns(unknown_patterns); + ASSERT_THAT(activation.SetUnknownAttributePatterns(unknown_patterns), IsOk()); ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr.Evaluate(activation, &arena_)); ASSERT_TRUE(result.IsUnknownSet()); @@ -1508,9 +1513,10 @@ TEST_F(DirectSelectStepTest, MissingAttributesToErrors) { TestAllTypes message; message.set_single_int64(1); activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - activation.SetMissingPatterns({cel::AttributePattern( - "test_all_types", - {cel::AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT(activation.SetMissingPatterns({cel::AttributePattern( + "test_all_types", + {cel::AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); ExecutionFrameBase frame(activation, options, type_provider_, cel::internal::GetTestingDescriptorPool(), @@ -1539,9 +1545,10 @@ TEST_F(DirectSelectStepTest, IdentifiesUnknowns) { TestAllTypes message; message.set_single_int64(1); activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - activation.SetUnknownPatterns({cel::AttributePattern( - "test_all_types", - {cel::AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( + "test_all_types", + {cel::AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); ExecutionFrameBase frame(activation, options, type_provider_, cel::internal::GetTestingDescriptorPool(), diff --git a/eval/public/BUILD b/eval/public/BUILD index f432767fa..39ab6368d 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -185,6 +185,7 @@ cc_library( ":cel_value", ":cel_value_producer", "//runtime/internal:attribute_matcher", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", diff --git a/eval/public/activation.h b/eval/public/activation.h index 6f2bb59c1..e9d7740af 100644 --- a/eval/public/activation.h +++ b/eval/public/activation.h @@ -6,8 +6,10 @@ #include #include +#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "eval/public/base_activation.h" @@ -77,9 +79,10 @@ class Activation : public BaseActivation { // // If a field access is found to match any of the provided patterns, the // result is treated as a missing attribute error. - void set_missing_attribute_patterns( + absl::Status SetMissingAttributePatterns( std::vector missing_attribute_patterns) { missing_attribute_patterns_ = std::move(missing_attribute_patterns); + return absl::OkStatus(); } const std::vector& missing_attribute_patterns() @@ -87,11 +90,19 @@ class Activation : public BaseActivation { return missing_attribute_patterns_; } + ABSL_DEPRECATED("Use SetUnknownAttributePatterns") + void set_unknown_attribute_patterns( + std::vector unknown_attribute_patterns) { + SetUnknownAttributePatterns(std::move(unknown_attribute_patterns)) + .IgnoreError(); + } + // Sets the collection of attribute patterns that will be recognized as // "unknown" values during expression evaluation. - void set_unknown_attribute_patterns( + absl::Status SetUnknownAttributePatterns( std::vector unknown_attribute_patterns) { unknown_attribute_patterns_ = std::move(unknown_attribute_patterns); + return absl::OkStatus(); } // Return the collection of attribute patterns that determine "unknown" diff --git a/eval/public/activation_test.cc b/eval/public/activation_test.cc index 238caf45e..33a0dfafe 100644 --- a/eval/public/activation_test.cc +++ b/eval/public/activation_test.cc @@ -20,6 +20,7 @@ namespace runtime { namespace { +using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::extensions::ProtoMemoryManager; using ::cel::expr::Expr; @@ -226,7 +227,8 @@ TEST(ActivationTest, ErrorPathTest) { CelAttributePattern::MatchType::FULL); EXPECT_TRUE(activation.missing_attribute_patterns().empty()); - activation.set_missing_attribute_patterns({destination_ip_pattern}); + ASSERT_THAT(activation.SetMissingAttributePatterns({destination_ip_pattern}), + IsOk()); EXPECT_EQ( activation.missing_attribute_patterns()[0].IsMatch(trail.attribute()), CelAttributePattern::MatchType::FULL); diff --git a/eval/tests/unknowns_end_to_end_test.cc b/eval/tests/unknowns_end_to_end_test.cc index b8132e42f..eaa9c163a 100644 --- a/eval/tests/unknowns_end_to_end_test.cc +++ b/eval/tests/unknowns_end_to_end_test.cc @@ -168,7 +168,9 @@ TEST_F(UnknownsTest, NoUnknowns) { TEST_F(UnknownsTest, UnknownAttributes) { PrepareBuilder(UnknownProcessingOptions::kAttributeOnly); - activation_.set_unknown_attribute_patterns({CelAttributePattern("var1", {})}); + ASSERT_THAT(activation_.SetUnknownAttributePatterns( + {CelAttributePattern("var1", {})}), + IsOk()); activation_.InsertValue("var2", CelValue::CreateInt64(3)); ASSERT_THAT(activation_.InsertFunction(std::make_unique( "F1", FunctionResponse::kTrue)), @@ -193,7 +195,9 @@ TEST_F(UnknownsTest, UnknownAttributes) { TEST_F(UnknownsTest, UnknownAttributesPruning) { PrepareBuilder(UnknownProcessingOptions::kAttributeOnly); - activation_.set_unknown_attribute_patterns({CelAttributePattern("var1", {})}); + ASSERT_THAT(activation_.SetUnknownAttributePatterns( + {CelAttributePattern("var1", {})}), + IsOk()); activation_.InsertValue("var2", CelValue::CreateInt64(5)); ASSERT_THAT(activation_.InsertFunction(std::make_unique( "F1", FunctionResponse::kTrue)), @@ -310,7 +314,9 @@ TEST_F(UnknownsTest, UnknownFunctions) { TEST_F(UnknownsTest, UnknownsMerge) { PrepareBuilder(UnknownProcessingOptions::kAttributeAndFunction); activation_.InsertValue("var1", CelValue::CreateInt64(5)); - activation_.set_unknown_attribute_patterns({CelAttributePattern("var2", {})}); + ASSERT_THAT(activation_.SetUnknownAttributePatterns( + {CelAttributePattern("var2", {})}), + IsOk()); ASSERT_THAT(activation_.InsertFunction(std::make_unique( "F1", FunctionResponse::kUnknown)), @@ -725,12 +731,15 @@ TEST(UnknownsIterAttrTest, IterAttributeTrail) { activation.InsertValue("var", CelProtoWrapper::CreateMessage(&list, &arena)); // var[1]['elem1'] is unknown - activation.set_unknown_attribute_patterns({CelAttributePattern( - "var", { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), - CreateCelAttributeQualifierPattern( - CelValue::CreateStringView("elem1")), - })}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("elem1")), + })}), + IsOk()); ASSERT_THAT(activation.InsertFunction(std::make_unique( "Fn", FunctionResponse::kFalse, CelValue::Type::kMap)), @@ -920,12 +929,15 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailMap) { activation.InsertValue("var", CelProtoWrapper::CreateMessage(&list, &arena)); // var[1]['key'] is unknown - activation.set_unknown_attribute_patterns({CelAttributePattern( - "var", - { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), - CreateCelAttributeQualifierPattern(CelValue::CreateStringView("key")), - })}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("key")), + })}), + IsOk()); ASSERT_THAT(activation.InsertFunction(std::make_unique( "Fn", FunctionResponse::kFalse, CelValue::Type::kDouble)), @@ -1041,8 +1053,9 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailExact) { unknown_attribute_patterns.push_back(CelAttributePattern( "list_var", {CreateCelAttributeQualifierPattern(CelValue::CreateInt64(0))})); - activation.set_unknown_attribute_patterns( - std::move(unknown_attribute_patterns)); + ASSERT_THAT(activation.SetUnknownAttributePatterns( + std::move(unknown_attribute_patterns)), + IsOk()); ASSERT_OK_AND_ASSIGN( auto plan, builder->CreateExpression(&expr.expr(), &expr.source_info())); @@ -1087,12 +1100,15 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailFilterValues) { activation.InsertValue("var", CelProtoWrapper::CreateMessage(&list, &arena)); // var[1]['value_key'] is unknown - activation.set_unknown_attribute_patterns({CelAttributePattern( - "var", { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), - CreateCelAttributeQualifierPattern( - CelValue::CreateStringView("value_key")), - })}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("value_key")), + })}), + IsOk()); auto plan = builder->CreateExpression(&expr, nullptr).value(); CelValue response = plan->Evaluate(activation, &arena).value(); @@ -1136,21 +1152,23 @@ TEST(UnknownsIterAttrTest, IterAttributeTrailFilterConditions) { activation.InsertValue("var", CelProtoWrapper::CreateMessage(&list, &arena)); // var[1]['value_key'] is unknown - activation.set_unknown_attribute_patterns( - {CelAttributePattern( - "var", - { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), - CreateCelAttributeQualifierPattern( - CelValue::CreateStringView("filter_key")), - }), - CelAttributePattern( - "var", - { - CreateCelAttributeQualifierPattern(CelValue::CreateInt64(0)), - CreateCelAttributeQualifierPattern( - CelValue::CreateStringView("filter_key")), - })}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns( + {CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(1)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("filter_key")), + }), + CelAttributePattern( + "var", + { + CreateCelAttributeQualifierPattern(CelValue::CreateInt64(0)), + CreateCelAttributeQualifierPattern( + CelValue::CreateStringView("filter_key")), + })}), + IsOk()); auto plan = builder->CreateExpression(&expr, nullptr).value(); CelValue response = plan->Evaluate(activation, &arena).value(); diff --git a/extensions/bindings_ext_test.cc b/extensions/bindings_ext_test.cc index c8b12c24a..3c396616c 100644 --- a/extensions/bindings_ext_test.cc +++ b/extensions/bindings_ext_test.cc @@ -624,9 +624,10 @@ TEST_P(BindingsExtInteractionsTest, UnknownAttributesSelectOptimization) { ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr)); Arena arena; Activation activation; - activation.set_unknown_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child")})}); + ASSERT_THAT(activation.SetUnknownAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -662,11 +663,13 @@ TEST_P(BindingsExtInteractionsTest, ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr)); Arena arena; Activation activation; - activation.set_unknown_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -703,11 +706,13 @@ TEST_P(BindingsExtInteractionsTest, MissingAttributesSelectOptimization) { ASSERT_OK_AND_ASSIGN(auto cel_expr, builder->CreateExpression(&expr)); Arena arena; Activation activation; - activation.set_missing_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetMissingAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -751,10 +756,12 @@ TEST_P(BindingsExtInteractionsTest, UnknownAttribute) { &expr.expr(), &expr.source_info())); Arena arena; Activation activation; - activation.set_unknown_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -797,10 +804,12 @@ TEST_P(BindingsExtInteractionsTest, UnknownAttributeReturnValue) { &expr.expr(), &expr.source_info())); Arena arena; Activation activation; - activation.set_unknown_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetUnknownAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); @@ -847,10 +856,12 @@ TEST_P(BindingsExtInteractionsTest, MissingAttribute) { &expr.expr(), &expr.source_info())); Arena arena; Activation activation; - activation.set_missing_attribute_patterns({AttributePattern( - "msg", {AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - AttributeQualifierPattern::OfString("single_int64")})}); + ASSERT_THAT( + activation.SetMissingAttributePatterns({AttributePattern( + "msg", {AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + AttributeQualifierPattern::OfString("single_int64")})}), + IsOk()); NestedTestAllTypes msg; msg.mutable_child()->mutable_child()->mutable_payload()->set_single_int64(42); diff --git a/extensions/select_optimization_test.cc b/extensions/select_optimization_test.cc index ecc78ae9e..619db4a07 100644 --- a/extensions/select_optimization_test.cc +++ b/extensions/select_optimization_test.cc @@ -842,12 +842,13 @@ TEST_F(SelectOptimizationTest, MissingAttributeIndependentOfUnknown) { // activation only uses a ptr to the underlying message, persist them. NestedTestAllTypes var; - act.SetMissingPatterns( - {AttributePattern("nested_test_all_types", - { - AttributeQualifierPattern::OfString("child"), - AttributeQualifierPattern::OfString("payload"), - })}); + ASSERT_THAT(act.SetMissingPatterns({AttributePattern( + "nested_test_all_types", + { + AttributeQualifierPattern::OfString("child"), + AttributeQualifierPattern::OfString("payload"), + })}), + IsOk()); ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( R"pb( @@ -1268,10 +1269,9 @@ INSTANTIATE_TEST_SUITE_P( child { child { payload { single_int64: -42 } } } )pb"}}, [](google::protobuf::Arena*, Activation& act) { - act.SetUnknownPatterns({AttributePattern( + return act.SetUnknownPatterns({AttributePattern( "b", {AttributeQualifierPattern::OfString("child"), AttributeQualifierPattern::OfString("child")})}); - return absl::OkStatus(); }, [](const absl::StatusOr& got) { @@ -1297,11 +1297,10 @@ INSTANTIATE_TEST_SUITE_P( child { child { payload { single_int64: -42 } } } )pb"}}, [](google::protobuf::Arena*, Activation& act) { - act.SetUnknownPatterns({AttributePattern( + return act.SetUnknownPatterns({AttributePattern( "b", {AttributeQualifierPattern::OfString("child"), AttributeQualifierPattern::OfString("child"), AttributeQualifierPattern::OfString("child")})}); - return absl::OkStatus(); }, [](const absl::StatusOr& got) { @@ -1319,10 +1318,9 @@ INSTANTIATE_TEST_SUITE_P( child { child { payload { single_int64: -42 } } } )pb"}}, [](google::protobuf::Arena*, Activation& act) { - act.SetUnknownPatterns({ + return act.SetUnknownPatterns({ AttributePattern("b", {}), }); - return absl::OkStatus(); }, [](const absl::StatusOr& got) { @@ -1343,10 +1341,9 @@ INSTANTIATE_TEST_SUITE_P( child { child { payload { single_int64: -42 } } } )pb"}}, [](google::protobuf::Arena*, Activation& act) { - act.SetUnknownPatterns({ + return act.SetUnknownPatterns({ AttributePattern("a", {}), }); - return absl::OkStatus(); }, [](const absl::StatusOr& got) { ASSERT_OK_AND_ASSIGN(Value result, got); @@ -1363,12 +1360,11 @@ INSTANTIATE_TEST_SUITE_P( child { payload { standalone_message { bb: 20 } } } )pb"}}, [](google::protobuf::Arena*, Activation& act) { - act.SetMissingPatterns({AttributePattern( + return act.SetMissingPatterns({AttributePattern( "nested_test_all_types", { AttributeQualifierPattern::OfString("child"), })}); - return absl::OkStatus(); }, [](const absl::StatusOr& got) { @@ -1388,10 +1384,9 @@ INSTANTIATE_TEST_SUITE_P( child { payload { standalone_message { bb: 20 } } } )pb"}}, [](google::protobuf::Arena*, Activation& act) { - act.SetMissingPatterns({AttributePattern( + return act.SetMissingPatterns({AttributePattern( "b", {AttributeQualifierPattern::OfString("child"), AttributeQualifierPattern::OfString("child")})}); - return absl::OkStatus(); }, [](const absl::StatusOr& got) { diff --git a/runtime/BUILD b/runtime/BUILD index 84cf40e75..08e137669 100644 --- a/runtime/BUILD +++ b/runtime/BUILD @@ -78,6 +78,7 @@ cc_library( "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", diff --git a/runtime/activation.h b/runtime/activation.h index 8c4fb4073..0d647c32e 100644 --- a/runtime/activation.h +++ b/runtime/activation.h @@ -23,6 +23,7 @@ #include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/functional/any_invocable.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" @@ -97,20 +98,24 @@ class Activation final : public ActivationInterface { bool InsertOrAssignValueProvider(absl::string_view name, ValueProvider provider); - void AddUnknownPattern(cel::AttributePattern pattern) { + absl::Status AddUnknownPattern(cel::AttributePattern pattern) { unknown_patterns_.push_back(std::move(pattern)); + return absl::OkStatus(); } - void SetUnknownPatterns(std::vector patterns) { + absl::Status SetUnknownPatterns(std::vector patterns) { unknown_patterns_ = std::move(patterns); + return absl::OkStatus(); } - void AddMissingPattern(cel::AttributePattern pattern) { + absl::Status AddMissingPattern(cel::AttributePattern pattern) { missing_patterns_.push_back(std::move(pattern)); + return absl::OkStatus(); } - void SetMissingPatterns(std::vector patterns) { + absl::Status SetMissingPatterns(std::vector patterns) { missing_patterns_ = std::move(patterns); + return absl::OkStatus(); } // Returns true if the function was inserted (no other registered function has diff --git a/runtime/activation_test.cc b/runtime/activation_test.cc index 108ed7978..780258464 100644 --- a/runtime/activation_test.cc +++ b/runtime/activation_test.cc @@ -39,6 +39,7 @@ namespace cel { namespace { +using ::absl_testing::IsOk; using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using testing::ElementsAre; @@ -212,11 +213,13 @@ TEST_F(ActivationTest, ValuesAndProvidersShareNamespace) { TEST_F(ActivationTest, SetUnknownAttributes) { Activation activation; - activation.SetUnknownPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); + ASSERT_THAT( + activation.SetUnknownPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); EXPECT_THAT( activation.GetUnknownAttributes(), @@ -229,12 +232,14 @@ TEST_F(ActivationTest, SetUnknownAttributes) { TEST_F(ActivationTest, ClearUnknownAttributes) { Activation activation; - activation.SetUnknownPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); - activation.SetUnknownPatterns({}); + ASSERT_THAT( + activation.SetUnknownPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); + ASSERT_THAT(activation.SetUnknownPatterns({}), IsOk()); EXPECT_THAT(activation.GetUnknownAttributes(), IsEmpty()); } @@ -242,11 +247,13 @@ TEST_F(ActivationTest, ClearUnknownAttributes) { TEST_F(ActivationTest, SetMissingAttributes) { Activation activation; - activation.SetMissingPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); + ASSERT_THAT( + activation.SetMissingPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); EXPECT_THAT( activation.GetMissingAttributes(), @@ -259,12 +266,14 @@ TEST_F(ActivationTest, SetMissingAttributes) { TEST_F(ActivationTest, ClearMissingAttributes) { Activation activation; - activation.SetMissingPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); - activation.SetMissingPatterns({}); + ASSERT_THAT( + activation.SetMissingPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); + ASSERT_THAT(activation.SetMissingPatterns({}), IsOk()); EXPECT_THAT(activation.GetMissingAttributes(), IsEmpty()); } @@ -327,16 +336,20 @@ TEST_F(ActivationTest, MoveAssignment) { [](absl::string_view name, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, google::protobuf::Arena* absl_nonnull) -> absl::StatusOr> { return IntValue(42); })); - moved_from.SetUnknownPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); - moved_from.SetMissingPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); + ASSERT_THAT( + moved_from.SetUnknownPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); + ASSERT_THAT( + moved_from.SetMissingPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); Activation moved_to; moved_to = std::move(moved_from); @@ -378,16 +391,20 @@ TEST_F(ActivationTest, MoveCtor) { [](absl::string_view name, const google::protobuf::DescriptorPool* absl_nonnull, google::protobuf::MessageFactory* absl_nonnull, google::protobuf::Arena* absl_nonnull) -> absl::StatusOr> { return IntValue(42); })); - moved_from.SetUnknownPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); - moved_from.SetMissingPatterns( - {AttributePattern("var1", - {AttributeQualifierPattern::OfString("field1")}), - AttributePattern("var1", - {AttributeQualifierPattern::OfString("field2")})}); + ASSERT_THAT( + moved_from.SetUnknownPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); + ASSERT_THAT( + moved_from.SetMissingPatterns( + {AttributePattern("var1", + {AttributeQualifierPattern::OfString("field1")}), + AttributePattern("var1", + {AttributeQualifierPattern::OfString("field2")})}), + IsOk()); Activation moved_to = std::move(moved_from); From e6fb890eace1e3f524c4ee124dea23091cb9980d Mon Sep 17 00:00:00 2001 From: Justin King Date: Thu, 8 Oct 2026 15:07:41 -0700 Subject: [PATCH 276/350] Remove arena pointer from `SmallByteStringRep` Small string data is not actually owned by any arena, so preserving the arena pointer does not make sense. This allows us to have 23 bytes of inline storage instead of 15. PiperOrigin-RevId: 996097021 --- common/internal/byte_string.cc | 11 +++++------ common/internal/byte_string.h | 28 ++++++++-------------------- common/internal/byte_string_test.cc | 11 ++++++----- common/values/string_value.cc | 4 ---- 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/common/internal/byte_string.cc b/common/internal/byte_string.cc index 4daaa66a1..cbf28f422 100644 --- a/common/internal/byte_string.cc +++ b/common/internal/byte_string.cc @@ -37,7 +37,7 @@ ByteString ByteString::From(absl::string_view value, ABSL_DCHECK(arena != nullptr); ByteString result(UninitializedTag{}); if (value.size() <= kSmallByteStringCapacity) { - result.SetSmall(arena, value); + result.SetSmall(value); } else { char* arena_value = reinterpret_cast(arena->AllocateAligned(value.size())); @@ -52,7 +52,7 @@ ByteString ByteString::From(absl::Cord value, ABSL_DCHECK(arena != nullptr); ByteString result(UninitializedTag{}); if (value.size() <= kSmallByteStringCapacity) { - result.SetSmall(arena, value); + result.SetSmall(value); } else { result.SetLarge(arena, google::protobuf::Arena::Create(arena, std::move(value))); @@ -65,7 +65,7 @@ ByteString ByteString::From(std::string&& value, ABSL_DCHECK(arena != nullptr); ByteString result(UninitializedTag{}); if (value.size() <= kSmallByteStringCapacity) { - result.SetSmall(arena, value); + result.SetSmall(value); } else if (value.size() > sizeof(std::string)) { value.shrink_to_fit(); result.SetMedium( @@ -107,7 +107,6 @@ ByteString ByteString::Concat(const ByteString& lhs, const ByteString& rhs, // If the resulting string fits in inline storage, do it. result.rep_.header.kind = ByteStringKind::kSmall; result.rep_.small.size = result_size; - result.rep_.small.arena = arena; lhs.CopyToArray(result.rep_.small.data); rhs.CopyToArray(result.rep_.small.data + lhs_size); } else { @@ -221,7 +220,7 @@ ByteString ByteString::Substring(size_t pos, size_t npos) const { switch (GetKind()) { case ByteStringKind::kSmall: { ByteString result(UninitializedTag{}); - result.SetSmall(GetSmallArena(), GetSmall().substr(pos, npos - pos)); + result.SetSmall(GetSmall().substr(pos, npos - pos)); return result; } case ByteStringKind::kMedium: { @@ -420,7 +419,7 @@ ByteString ByteString::Clone(google::protobuf::Arena* absl_nonnull arena) const switch (GetKind()) { case ByteStringKind::kSmall: { ByteString result(UninitializedTag{}); - result.SetSmall(arena, GetSmall()); + result.SetSmall(GetSmall()); return result; } case ByteStringKind::kMedium: { diff --git a/common/internal/byte_string.h b/common/internal/byte_string.h index 6027d08ec..b9fb222a8 100644 --- a/common/internal/byte_string.h +++ b/common/internal/byte_string.h @@ -74,8 +74,7 @@ struct SmallByteStringRep final { #ifdef _MSC_VER #pragma pack(pop) #endif - char data[23 - sizeof(google::protobuf::Arena*)]; - google::protobuf::Arena* absl_nullable arena; + char data[23]; }; inline constexpr size_t kSmallByteStringCapacity = @@ -215,7 +214,7 @@ class [[nodiscard]] ByteString final { static ByteString Concat(const ByteString& lhs, const ByteString& rhs, google::protobuf::Arena* absl_nonnull arena); - ByteString() noexcept { SetSmallEmpty(nullptr); } + ByteString() noexcept { SetSmallEmpty(); } ByteString(const ByteString&) = default; ByteString(ByteString&&) = default; @@ -303,10 +302,12 @@ class [[nodiscard]] ByteString final { std::string* absl_nonnull scratch ABSL_ATTRIBUTE_LIFETIME_BOUND) const ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Returns the arena which owns the underling data for this byte string. + // Returns null when the data is not owned by an arena. google::protobuf::Arena* absl_nullable GetArena() const { switch (GetKind()) { case ByteStringKind::kSmall: - return GetSmallArena(); + return nullptr; case ByteStringKind::kMedium: return GetMediumArena(); case ByteStringKind::kLarge: @@ -373,16 +374,6 @@ class [[nodiscard]] ByteString final { return absl::string_view(rep.data, rep.size); } - google::protobuf::Arena* absl_nullable GetSmallArena() const { - ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kSmall); - return GetSmallArena(rep_.small); - } - - static google::protobuf::Arena* absl_nullable GetSmallArena( - const SmallByteStringRep& rep) { - return rep.arena; - } - google::protobuf::Arena* absl_nullable GetMediumArena() const { ABSL_DCHECK_EQ(GetKind(), ByteStringKind::kMedium); return GetMediumArena(rep_.medium); @@ -413,27 +404,24 @@ class [[nodiscard]] ByteString final { return rep.arena; } - void SetSmallEmpty(google::protobuf::Arena* absl_nullable arena) { + void SetSmallEmpty() { rep_.header.kind = ByteStringKind::kSmall; rep_.small.size = 0; - rep_.small.arena = arena; } - void SetSmall(google::protobuf::Arena* absl_nullable arena, absl::string_view string) { + void SetSmall(absl::string_view string) { ABSL_DCHECK_LE(string.size(), kSmallByteStringCapacity); rep_.header.kind = ByteStringKind::kSmall; rep_.small.size = string.size(); - rep_.small.arena = arena; if (!string.empty()) { std::memcpy(rep_.small.data, string.data(), rep_.small.size); } } - void SetSmall(google::protobuf::Arena* absl_nullable arena, const absl::Cord& cord) { + void SetSmall(const absl::Cord& cord) { ABSL_DCHECK_LE(cord.size(), kSmallByteStringCapacity); rep_.header.kind = ByteStringKind::kSmall; rep_.small.size = cord.size(); - rep_.small.arena = arena; CopyCordToArray(cord, rep_.small.data); } diff --git a/common/internal/byte_string_test.cc b/common/internal/byte_string_test.cc index 121158ada..b0cdee814 100644 --- a/common/internal/byte_string_test.cc +++ b/common/internal/byte_string_test.cc @@ -40,6 +40,7 @@ namespace { using ::testing::_; using ::testing::Eq; using ::testing::IsEmpty; +using ::testing::IsNull; using ::testing::Not; using ::testing::Optional; using ::testing::SizeIs; @@ -114,7 +115,7 @@ TEST_F(ByteStringTest, Default) { TEST_F(ByteStringTest, ConstructNullDataStringView) { ByteString byte_string = ByteString::From(absl::string_view(), GetArena()); EXPECT_THAT(byte_string, IsEmpty()); - EXPECT_EQ(byte_string.GetArena(), GetArena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } TEST_F(ByteStringTest, ConstructSmallCString) { @@ -124,7 +125,7 @@ TEST_F(ByteStringTest, ConstructSmallCString) { EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetArena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } TEST_F(ByteStringTest, ConstructMediumCString) { @@ -143,7 +144,7 @@ TEST_F(ByteStringTest, ConstructSmallRValueString) { EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetArena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } TEST_F(ByteStringTest, ConstructSmallLValueString) { @@ -153,7 +154,7 @@ TEST_F(ByteStringTest, ConstructSmallLValueString) { EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetArena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } TEST_F(ByteStringTest, ConstructMediumRValueString) { @@ -181,7 +182,7 @@ TEST_F(ByteStringTest, ConstructSmallCord) { EXPECT_THAT(byte_string, Not(IsEmpty())); EXPECT_EQ(byte_string, GetSmallStringView()); EXPECT_EQ(GetKind(byte_string), ByteStringKind::kSmall); - EXPECT_EQ(byte_string.GetArena(), GetArena()); + EXPECT_THAT(byte_string.GetArena(), IsNull()); } TEST_F(ByteStringTest, ConstructMediumOrLargeCord) { diff --git a/common/values/string_value.cc b/common/values/string_value.cc index 6aa7ac543..b9495fa69 100644 --- a/common/values/string_value.cc +++ b/common/values/string_value.cc @@ -616,7 +616,6 @@ Value StringValue::Substring(int64_t start, std::memcpy(result.value_.rep_.small.data, value_.rep_.small.data + *status_or_index, result.value_.rep_.small.size); - result.value_.rep_.small.arena = value_.rep_.small.arena; return result; } case common_internal::ByteStringKind::kMedium: { @@ -744,7 +743,6 @@ Value StringValue::Substring(int64_t start, int64_t end, std::memcpy(result.value_.rep_.small.data, value_.rep_.small.data + status_or_indices->first, result.value_.rep_.small.size); - result.value_.rep_.small.arena = value_.rep_.small.arena; return result; } case common_internal::ByteStringKind::kMedium: { @@ -1430,7 +1428,6 @@ Value StringValue::CharAt(int64_t pos, common_internal::ByteStringKind::kSmall; result.value_.rep_.small.size = cel::internal::Utf8Encode( code_point, result.value_.rep_.small.data); - result.value_.rep_.small.arena = value_.GetArena(); return result; } rep.remove_prefix(code_units); @@ -1460,7 +1457,6 @@ Value StringValue::CharAt(int64_t pos, common_internal::ByteStringKind::kSmall; result.value_.rep_.small.size = cel::internal::Utf8Encode( code_point, result.value_.rep_.small.data); - result.value_.rep_.small.arena = nullptr; return result; } absl::Cord::Advance(&begin, code_units); From fcad3d5b36e936d6587c0856f717e94debf8cca1 Mon Sep 17 00:00:00 2001 From: Justin King Date: Thu, 8 Oct 2026 15:41:07 -0700 Subject: [PATCH 277/350] Remove recursive evaluation PiperOrigin-RevId: 996115495 --- conformance/BUILD | 4 +- conformance/run.bzl | 45 +- conformance/run.cc | 4 - eval/compiler/BUILD | 33 +- .../cel_expression_builder_flat_impl.cc | 15 - .../cel_expression_builder_flat_impl_test.cc | 117 +--- eval/compiler/constant_folding.cc | 14 - eval/compiler/flat_expr_builder.cc | 497 ---------------- .../flat_expr_builder_comprehensions_test.cc | 43 +- eval/compiler/flat_expr_builder_extensions.cc | 105 ---- eval/compiler/flat_expr_builder_extensions.h | 70 +-- .../flat_expr_builder_extensions_test.cc | 58 +- eval/compiler/flat_expr_builder_test.cc | 142 +---- eval/compiler/qualified_reference_resolver.cc | 2 +- .../regex_precompilation_optimization.cc | 37 -- eval/eval/BUILD | 221 +------ eval/eval/cel_expression_flat_impl.cc | 63 -- eval/eval/cel_expression_flat_impl.h | 81 --- eval/eval/compiler_constant_step.cc | 31 - eval/eval/compiler_constant_step.h | 52 -- eval/eval/compiler_constant_step_test.cc | 73 --- eval/eval/comprehension_step.cc | 386 ------------ eval/eval/comprehension_step.h | 14 - eval/eval/comprehension_step_test.cc | 319 ---------- eval/eval/const_value_step.h | 23 - eval/eval/container_access_step.cc | 50 +- eval/eval/container_access_step.h | 9 +- eval/eval/container_access_step_test.cc | 12 +- eval/eval/create_list_step.cc | 124 +--- eval/eval/create_list_step.h | 18 +- eval/eval/create_list_step_test.cc | 302 +--------- eval/eval/create_map_step.cc | 137 +---- eval/eval/create_map_step.h | 19 +- eval/eval/create_map_step_test.cc | 35 +- eval/eval/create_struct_step.cc | 120 +--- eval/eval/create_struct_step.h | 10 +- eval/eval/create_struct_step_test.cc | 40 +- eval/eval/direct_expression_step.cc | 20 - eval/eval/direct_expression_step.h | 82 --- eval/eval/equality_steps.cc | 78 +-- eval/eval/equality_steps.h | 15 - eval/eval/equality_steps_test.cc | 143 +---- eval/eval/evaluator_core.cc | 16 - eval/eval/evaluator_core.h | 21 - eval/eval/function_step.cc | 95 --- eval/eval/function_step.h | 17 - eval/eval/function_step_test.cc | 193 ------ eval/eval/ident_step.cc | 65 --- eval/eval/ident_step.h | 12 - eval/eval/ident_step_test.cc | 114 ---- eval/eval/lazy_init_step.cc | 109 ---- eval/eval/lazy_init_step.h | 21 - eval/eval/logic_step.cc | 274 --------- eval/eval/logic_step.h | 24 - eval/eval/logic_step_test.cc | 379 ------------ eval/eval/optional_or_step.cc | 108 ---- eval/eval/optional_or_step.h | 11 +- eval/eval/optional_or_step_test.cc | 383 ------------ eval/eval/regex_match_step.cc | 45 +- eval/eval/regex_match_step.h | 8 +- eval/eval/regex_match_step_test.cc | 1 - eval/eval/select_step.cc | 121 +--- eval/eval/select_step.h | 10 +- eval/eval/select_step_test.cc | 552 ------------------ eval/eval/shadowable_value_step.cc | 43 +- eval/eval/shadowable_value_step.h | 7 +- eval/eval/shadowable_value_step_test.cc | 6 +- eval/eval/ternary_step.cc | 117 +--- eval/eval/ternary_step.h | 12 +- eval/eval/ternary_step_test.cc | 194 +----- eval/eval/trace_step.h | 73 --- eval/public/BUILD | 3 - eval/public/cel_options.h | 8 +- extensions/BUILD | 2 - extensions/select_optimization.cc | 67 --- runtime/BUILD | 2 - runtime/internal/BUILD | 5 - runtime/internal/runtime_impl.cc | 76 --- runtime/internal/runtime_impl.h | 5 - runtime/optional_types_test.cc | 36 -- runtime/runtime_options.h | 8 +- .../standard_runtime_builder_factory_test.cc | 201 +------ 82 files changed, 108 insertions(+), 6999 deletions(-) delete mode 100644 eval/eval/compiler_constant_step.cc delete mode 100644 eval/eval/compiler_constant_step.h delete mode 100644 eval/eval/compiler_constant_step_test.cc delete mode 100644 eval/eval/const_value_step.h delete mode 100644 eval/eval/direct_expression_step.cc delete mode 100644 eval/eval/direct_expression_step.h delete mode 100644 eval/eval/optional_or_step_test.cc delete mode 100644 eval/eval/trace_step.h diff --git a/conformance/BUILD b/conformance/BUILD index b4ac3718f..a6bf15f6b 100644 --- a/conformance/BUILD +++ b/conformance/BUILD @@ -246,7 +246,7 @@ _TESTS_TO_SKIP_LEGACY_DASHBOARD = [ ] # Generates a bunch of `cc_test` whose names follow the pattern -# `conformance_(...)_{pratt|antlr}_{optimized|unoptimized}_{recursive|iterative}`. +# `conformance_(...)_{pratt|antlr}_{optimized|unoptimized}`. gen_conformance_tests( name = "conformance_parse_only", data = _ALL_TESTS, @@ -315,7 +315,7 @@ gen_conformance_tests( ) # Generates a bunch of `cc_test` whose names follow the pattern -# `conformance_dashboard_..._{pratt|antlr}_{optimized|unoptimized}_{recursive|iterative}`. +# `conformance_dashboard_..._{pratt|antlr}_{optimized|unoptimized}`. gen_conformance_tests( name = "conformance_dashboard_parse_only", dashboard = True, diff --git a/conformance/run.bzl b/conformance/run.bzl index 76b33efb3..9c4b315e5 100644 --- a/conformance/run.bzl +++ b/conformance/run.bzl @@ -47,17 +47,16 @@ def _expand_tests_to_skip(tests_to_skip): result.append(test_to_skip[0:slash] + part) return result -def _conformance_test_name(name, pratt, optimize, recursive): +def _conformance_test_name(name, pratt, optimize): return "_".join( [ name, "pratt" if pratt else "antlr", "optimized" if optimize else "unoptimized", - "recursive" if recursive else "iterative", ], ) -def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators, pratt): +def _conformance_test_args(modern, optimize, select_opt, skip_check, dashboard, enable_variadic_logical_operators, pratt): args = [] if modern: args.append("--modern") @@ -65,8 +64,6 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, args.append("--opt") if select_opt: args.append("--select_optimization") - if recursive: - args.append("--recursive") if skip_check: args.append("--skip_check") else: @@ -81,10 +78,10 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, args.append("--noenable_pratt_parser") return args -def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard, enable_variadic_logical_operators, pratt): +def _conformance_test(name, data, modern, optimize, select_opt, skip_check, skip_tests, tags, dashboard, enable_variadic_logical_operators, pratt): cc_test( - name = _conformance_test_name(name, pratt, optimize, recursive), - args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators, pratt) + ["$(rlocationpath {})".format(test) for test in data], + name = _conformance_test_name(name, pratt, optimize), + args = _conformance_test_args(modern, optimize, select_opt, skip_check, dashboard, enable_variadic_logical_operators, pratt) + ["$(rlocationpath {})".format(test) for test in data], env = select( { "@platforms//os:windows": {"CEL_SKIP_TESTS": ",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS)}, @@ -115,23 +112,21 @@ def gen_conformance_tests(name, data, modern = False, checked = False, select_op tests = [] for pratt in (True, False): for optimize in (True, False): - for recursive in (True, False): - test_name = _conformance_test_name(name, pratt, optimize, recursive) - tests.append(test_name) - _conformance_test( - name, - data, - modern = modern, - optimize = optimize, - recursive = recursive, - select_opt = select_opt, - skip_check = skip_check, - skip_tests = _expand_tests_to_skip(skip_tests), - tags = tags, - dashboard = dashboard, - enable_variadic_logical_operators = enable_variadic_logical_operators, - pratt = pratt, - ) + test_name = _conformance_test_name(name, pratt, optimize) + tests.append(test_name) + _conformance_test( + name, + data, + modern = modern, + optimize = optimize, + select_opt = select_opt, + skip_check = skip_check, + skip_tests = _expand_tests_to_skip(skip_tests), + tags = tags, + dashboard = dashboard, + enable_variadic_logical_operators = enable_variadic_logical_operators, + pratt = pratt, + ) native.test_suite( name = name, tests = tests, diff --git a/conformance/run.cc b/conformance/run.cc index 6b7bae071..72eeeb4dd 100644 --- a/conformance/run.cc +++ b/conformance/run.cc @@ -59,9 +59,6 @@ ABSL_FLAG(bool, opt, false, "Enable optimizations (constant folding)"); ABSL_FLAG( bool, modern, false, "Use modern cel::Value APIs implementation of the conformance service."); -ABSL_FLAG(bool, recursive, false, - "Enable recursive plans. Depth limited to slightly more than the " - "default nesting limit."); ABSL_FLAG(std::vector, skip_tests, {}, "Tests to skip"); ABSL_FLAG(bool, dashboard, false, "Dashboard mode, ignore test failures"); ABSL_FLAG(bool, skip_check, true, "Skip type checking the expressions"); @@ -264,7 +261,6 @@ NewConformanceServiceFromFlags() { cel_conformance::ConformanceServiceOptions{ .optimize = absl::GetFlag(FLAGS_opt), .modern = absl::GetFlag(FLAGS_modern), - .recursive = absl::GetFlag(FLAGS_recursive), .select_optimization = absl::GetFlag(FLAGS_select_optimization), .enable_variadic_logical_operators = absl::GetFlag(FLAGS_enable_variadic_logical_operators), diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 89677776c..d75e6e50d 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -32,13 +32,9 @@ cc_library( "//base:ast", "//base:data", "//common:expr", - "//common:native_type", "//common:value", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/eval:expression_step_logic", - "//eval/eval:trace_step", - "//internal:casts", "//runtime:runtime_options", "//runtime/internal:issue_collector", "//runtime/internal:runtime_env", @@ -51,7 +47,6 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:variant", "@com_google_protobuf//:protobuf", ], @@ -64,13 +59,8 @@ cc_test( ":flat_expr_builder_extensions", ":resolver", "//common:expr", - "//common:native_type", - "//common:value", - "//eval/eval:const_value_step", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", - "//eval/eval:function_step", - "//internal:status_macros", + "//eval/eval:expression_step_logic", "//internal:testing", "//runtime:function_registry", "//runtime:runtime_issue", @@ -102,7 +92,6 @@ cc_library( "//base:ast", "//base:builtins", "//base:data", - "//common:allocator", "//common:ast", "//common:ast_traverse", "//common:ast_visitor", @@ -112,22 +101,16 @@ cc_library( "//common:type", "//common:type_spec_resolver", "//common:value", - "//eval/eval:const_value_step", "//eval/eval:container_access_step", "//eval/eval:create_list_step", "//eval/eval:create_map_step", "//eval/eval:create_struct_step", - "//eval/eval:direct_expression_step", - "//eval/eval:equality_steps", "//eval/eval:evaluator_core", "//eval/eval:expression_step_logic", - "//eval/eval:function_step", - "//eval/eval:ident_step", "//eval/eval:optional_or_step", "//eval/eval:select_step", "//eval/eval:shadowable_value_step", "//eval/eval:ternary_step", - "//eval/eval:trace_step", "//internal:status_macros", "//runtime:function_registry", "//runtime:runtime_issue", @@ -137,7 +120,6 @@ cc_library( "//runtime/internal:issue_collector", "//runtime/internal:runtime_env", "@com_google_absl//absl/algorithm:container", - "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -171,7 +153,6 @@ cc_test( "//eval/public:activation", "//eval/public:builtin_func_registrar", "//eval/public:cel_attribute", - "//eval/public:cel_builtins", "//eval/public:cel_expr_builder_factory", "//eval/public:cel_expression", "//eval/public:cel_function", @@ -197,7 +178,6 @@ cc_test( "//runtime:runtime_options", "//runtime:standard_functions", "//runtime/internal:runtime_env_testing", - "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", @@ -255,9 +235,7 @@ cc_library( deps = [ ":flat_expr_builder", "//base:ast", - "//common:native_type", "//eval/eval:cel_expression_flat_impl", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/public:cel_expression", "//eval/public:cel_function_registry", @@ -286,9 +264,6 @@ cc_test( ], deps = [ ":cel_expression_builder_flat_impl", - ":constant_folding", - ":regex_precompilation_optimization", - "//eval/eval:cel_expression_flat_impl", "//eval/public:activation", "//eval/public:builtin_func_registrar", "//eval/public:cel_expression", @@ -297,7 +272,6 @@ cc_test( "//eval/public:portable_cel_function_adapter", "//eval/public/containers:container_backed_map_impl", "//eval/public/structs:cel_proto_wrapper", - "//eval/public/structs:protobuf_descriptor_type_provider", "//eval/public/testing:matchers", "//extensions:bindings_ext", "//internal:status_macros", @@ -334,12 +308,10 @@ cc_library( "//common:constant", "//common:expr", "//common:value", - "//eval/eval:const_value_step", "//eval/eval:evaluator_core", "//internal:status_macros", "//runtime:activation", "//runtime/internal:convert_constant", - "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -407,7 +379,6 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", ], ) @@ -547,8 +518,6 @@ cc_library( "//common:expr", "//common:native_type", "//common:value", - "//eval/eval:compiler_constant_step", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/eval:regex_match_step", "//internal:re2_options", diff --git a/eval/compiler/cel_expression_builder_flat_impl.cc b/eval/compiler/cel_expression_builder_flat_impl.cc index 830b5ff1d..f519d9393 100644 --- a/eval/compiler/cel_expression_builder_flat_impl.cc +++ b/eval/compiler/cel_expression_builder_flat_impl.cc @@ -27,9 +27,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "base/ast.h" -#include "common/native_type.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/public/cel_expression.h" #include "extensions/protobuf/ast_converters.h" @@ -96,19 +94,6 @@ CelExpressionBuilderFlatImpl::CreateExpressionImpl( warnings->push_back(issue.ToStatus()); } } - if (flat_expr_builder_.options().max_recursion_depth != 0 && - !impl.subexpressions().empty() && - // mainline expression is exactly one recursive step. - impl.subexpressions().front().size() == 1 && - impl.subexpressions().front().front().IsGenericStep() && - impl.subexpressions() - .front() - .front() - .GetGenericStep() - ->GetNativeTypeId() == - cel::NativeTypeId::For()) { - return CelExpressionRecursiveImpl::Create(env_, std::move(impl)); - } return std::make_unique(env_, std::move(impl)); } diff --git a/eval/compiler/cel_expression_builder_flat_impl_test.cc b/eval/compiler/cel_expression_builder_flat_impl_test.cc index 9802d2a05..b48fbd959 100644 --- a/eval/compiler/cel_expression_builder_flat_impl_test.cc +++ b/eval/compiler/cel_expression_builder_flat_impl_test.cc @@ -30,9 +30,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "eval/compiler/constant_folding.h" -#include "eval/compiler/regex_precompilation_optimization.h" -#include "eval/eval/cel_expression_flat_impl.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_expression.h" @@ -72,8 +69,6 @@ using ::google::api::expr::parser::ParseWithMacros; using ::testing::_; using ::testing::Contains; using ::testing::HasSubstr; -using ::testing::IsNull; -using ::testing::NotNull; TEST(CelExpressionBuilderFlatImplTest, Error) { Expr expr; @@ -100,14 +95,14 @@ TEST(CelExpressionBuilderFlatImplTest, ParsedExpr) { EXPECT_THAT(result, test::IsCelInt64(3)); } -struct RecursiveTestCase { +struct TestCase { std::string test_name; std::string expr; test::CelValueMatcher matcher; std::string pb_expr; }; -class RecursivePlanTest : public ::testing::TestWithParam { +class PlanTest : public ::testing::TestWithParam { protected: absl::Status SetupBuilder(CelExpressionBuilderFlatImpl& builder) { builder.GetTypeRegistry()->RegisterEnum("TestEnum", @@ -146,7 +141,7 @@ class RecursivePlanTest : public ::testing::TestWithParam { } }; -absl::StatusOr ParseTestCase(const RecursiveTestCase& test_case) { +absl::StatusOr ParseTestCase(const TestCase& test_case) { static const std::vector* kMacros = []() { auto* result = new std::vector(Macro::AllMacros()); absl::c_copy(cel::extensions::bindings_macros(), @@ -166,107 +161,14 @@ absl::StatusOr ParseTestCase(const RecursiveTestCase& test_case) { return absl::InvalidArgumentError("No expression provided"); } -TEST_P(RecursivePlanTest, ParsedExprRecursiveImpl) { - const RecursiveTestCase& test_case = GetParam(); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, ParseTestCase(test_case)); - cel::RuntimeOptions options; - options.container = "cel.expr.conformance.proto3"; - google::protobuf::Arena arena; - // Unbounded. - options.max_recursion_depth = -1; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - - ASSERT_OK(SetupBuilder(builder)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr plan, - builder.CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - - EXPECT_THAT(dynamic_cast(plan.get()), - NotNull()); - - Activation activation; - - ASSERT_OK(SetupActivation(activation, &arena)); - - ASSERT_OK_AND_ASSIGN(CelValue result, plan->Evaluate(activation, &arena)); - EXPECT_THAT(result, test_case.matcher); -} - -TEST_P(RecursivePlanTest, ParsedExprRecursiveOptimizedImpl) { - const RecursiveTestCase& test_case = GetParam(); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, ParseTestCase(test_case)); - cel::RuntimeOptions options; - options.container = "cel.expr.conformance.proto3"; - google::protobuf::Arena arena; - // Unbounded. - options.max_recursion_depth = -1; - options.enable_comprehension_list_append = true; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - - ASSERT_OK(SetupBuilder(builder)); - - builder.flat_expr_builder().AddProgramOptimizer( - cel::runtime_internal::CreateConstantFoldingOptimizer()); - builder.flat_expr_builder().AddProgramOptimizer( - CreateRegexPrecompilationExtension(options.regex_max_program_size)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr plan, - builder.CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - - EXPECT_THAT(dynamic_cast(plan.get()), - NotNull()); - - Activation activation; - - ASSERT_OK(SetupActivation(activation, &arena)); - - ASSERT_OK_AND_ASSIGN(CelValue result, plan->Evaluate(activation, &arena)); - EXPECT_THAT(result, test_case.matcher); -} - -TEST_P(RecursivePlanTest, ParsedExprRecursiveTraceSupport) { - const RecursiveTestCase& test_case = GetParam(); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, ParseTestCase(test_case)); - cel::RuntimeOptions options; - options.container = "cel.expr.conformance.proto3"; - google::protobuf::Arena arena; - auto cb = [](int64_t id, const CelValue& value, google::protobuf::Arena* arena) { - return absl::OkStatus(); - }; - // Unbounded. - options.max_recursion_depth = -1; - options.enable_recursive_tracing = true; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - - ASSERT_OK(SetupBuilder(builder)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr plan, - builder.CreateExpression(&parsed_expr.expr(), - &parsed_expr.source_info())); - - EXPECT_THAT(dynamic_cast(plan.get()), - NotNull()); - - Activation activation; - - ASSERT_OK(SetupActivation(activation, &arena)); - - ASSERT_OK_AND_ASSIGN(CelValue result, plan->Trace(activation, &arena, cb)); - EXPECT_THAT(result, test_case.matcher); -} - -TEST_P(RecursivePlanTest, Disabled) { +TEST_P(PlanTest, Basic) { google::protobuf::LinkMessageReflection(); - const RecursiveTestCase& test_case = GetParam(); + const TestCase& test_case = GetParam(); ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, ParseTestCase(test_case)); cel::RuntimeOptions options; options.container = "cel.expr.conformance.proto3"; google::protobuf::Arena arena; - // disabled. - options.max_recursion_depth = 0; CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); ASSERT_OK(SetupBuilder(builder)); @@ -275,9 +177,6 @@ TEST_P(RecursivePlanTest, Disabled) { builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info())); - EXPECT_THAT(dynamic_cast(plan.get()), - IsNull()); - Activation activation; ASSERT_OK(SetupActivation(activation, &arena)); @@ -287,8 +186,8 @@ TEST_P(RecursivePlanTest, Disabled) { } INSTANTIATE_TEST_SUITE_P( - RecursivePlanTest, RecursivePlanTest, - testing::ValuesIn(std::vector{ + PlanTest, PlanTest, + testing::ValuesIn(std::vector{ {"constant", "'abc'", test::IsCelString("abc")}, {"call", "1 + 2", test::IsCelInt64(3)}, {"nested_call", "1 + 1 + 1 + 1", test::IsCelInt64(4)}, @@ -545,7 +444,7 @@ INSTANTIATE_TEST_SUITE_P( } })pb"}}), - [](const testing::TestParamInfo& info) -> std::string { + [](const testing::TestParamInfo& info) -> std::string { return info.param.test_name; }); diff --git a/eval/compiler/constant_folding.cc b/eval/compiler/constant_folding.cc index 071f16186..fb58d7d48 100644 --- a/eval/compiler/constant_folding.cc +++ b/eval/compiler/constant_folding.cc @@ -19,7 +19,6 @@ #include #include -#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -31,7 +30,6 @@ #include "common/value.h" #include "eval/compiler/flat_expr_builder_extensions.h" #include "eval/compiler/resolver.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/evaluator_core.h" #include "internal/status_macros.h" #include "runtime/activation.h" @@ -49,7 +47,6 @@ using ::cel::builtin::kAnd; using ::cel::builtin::kOr; using ::cel::builtin::kTernary; using ::cel::runtime_internal::ConvertConstant; -using ::google::api::expr::runtime::CreateConstValueDirectStep; using ::google::api::expr::runtime::EvaluationListener; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionPath; @@ -229,17 +226,6 @@ absl::Status ConstantFoldingExtension::OnPostVisit(PlannerContext& context, } } - // If recursive planning enabled (recursion limit unbounded or at least 1), - // use a recursive (direct) step for the folded constant. - // - // Constant folding is applied leaf to root based on the program plan so far, - // so the planner will have an opportunity to validate that the recursion - // limit is being followed when visiting parent nodes in the AST. - if (context.options().max_recursion_depth != 0) { - return context.ReplaceSubplan( - node, CreateConstValueDirectStep(std::move(value), node.id()), 1); - } - // Otherwise make a stack machine plan. ExecutionPath new_plan; new_plan.push_back(ExpressionStep::MakeConstant(value, node.id())); diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 5c8d7c36a..a305dec17 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -50,7 +50,6 @@ #include "base/ast.h" #include "base/builtins.h" #include "base/type_provider.h" -#include "common/allocator.h" #include "common/ast.h" #include "common/ast_traverse.h" #include "common/ast_visitor.h" @@ -64,24 +63,17 @@ #include "eval/compiler/flat_expr_builder_extensions.h" #include "eval/compiler/resolver.h" #include "eval/eval/comprehension_step.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/container_access_step.h" #include "eval/eval/create_list_step.h" #include "eval/eval/create_map_step.h" #include "eval/eval/create_struct_step.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/equality_steps.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_logic.h" #include "eval/eval/function_step.h" -#include "eval/eval/ident_step.h" -#include "eval/eval/lazy_init_step.h" -#include "eval/eval/logic_step.h" #include "eval/eval/optional_or_step.h" #include "eval/eval/select_step.h" #include "eval/eval/shadowable_value_step.h" #include "eval/eval/ternary_step.h" -#include "eval/eval/trace_step.h" #include "internal/status_macros.h" #include "runtime/internal/convert_constant.h" #include "runtime/internal/issue_collector.h" @@ -97,7 +89,6 @@ namespace { using ::cel::Ast; using ::cel::AstTraverse; using ::cel::RuntimeIssue; -using ::cel::StringValue; using ::cel::Value; using ::cel::runtime_internal::ConvertConstant; using ::cel::runtime_internal::GetLegacyRuntimeTypeProvider; @@ -111,13 +102,6 @@ constexpr absl::string_view kBlock = "cel.@block"; // Forward declare to resolve circular dependency for short_circuiting visitors. class FlatExprVisitor; -// Error code for failed recursive program building. Generally indicates an -// optimization doesn't support recursive programs. -absl::Status FailedRecursivePlanning() { - return absl::InternalError( - "failed to build recursive program. check for unsupported optimizations"); -} - // Helper for bookkeeping variables mapped to indexes. class IndexManager { public: @@ -466,17 +450,6 @@ class ComprehensionVisitor { size_t accu_slot_; }; -absl::flat_hash_set MakeOptionalIndicesSet( - const cel::ListExpr& create_list_expr) { - absl::flat_hash_set optional_indices; - for (size_t i = 0; i < create_list_expr.elements().size(); ++i) { - if (create_list_expr.elements()[i].optional()) { - optional_indices.insert(static_cast(i)); - } - } - return optional_indices; -} - absl::flat_hash_set MakeOptionalIndicesSet( const cel::StructExpr& create_struct_expr) { absl::flat_hash_set optional_indices; @@ -590,12 +563,6 @@ class FlatExprVisitor : public cel::AstVisitor { } } - void SetMaxRecursionDepth(int max_recursion_depth) { - max_recursion_depth_ = max_recursion_depth; - } - - bool PlanRecursiveProgram() const { return max_recursion_depth_ > 0; } - void SetResolvedType(const cel::Expr& expr, cel::Type type) { resolved_types_[&expr] = std::move(type); } @@ -667,14 +634,6 @@ class FlatExprVisitor : public cel::AstVisitor { } } - auto* subexpression = program_builder_.current(); - if (subexpression != nullptr && options_.enable_recursive_tracing && - subexpression->IsRecursive()) { - auto program = subexpression->ExtractRecursiveProgram(); - subexpression->set_recursive_program( - std::make_unique(std::move(program.step)), program.depth); - } - program_builder_.ExitSubexpression(&expr); if (!comprehension_stack_.empty() && @@ -713,13 +672,6 @@ class FlatExprVisitor : public cel::AstVisitor { return; } - if (options_.max_recursion_depth > 0 || options_.max_recursion_depth < 0) { - SetRecursiveStep(CreateConstValueDirectStep( - std::move(converted_value).value(), expr.id()), - 1); - return; - } - AddStep(ExpressionStep::MakeConstant(std::move(converted_value).value(), expr.id())); } @@ -838,25 +790,12 @@ class FlatExprVisitor : public cel::AstVisitor { absl::InternalError("bad subexpression reference")); return; } - if (subexpression->IsRecursive()) { - const auto& program = subexpression->recursive_program(); - SetRecursiveStep( - CreateDirectLazyInitStep(slot.slot, program.step.get(), expr.id()), - program.depth + 1); - } else { // Off by one since mainline expression will be index 0. AddStep(ExpressionStep::MakeLazyInitStep( slot.slot, slot.subexpression + 1, expr.id())); - } return; } else if (slot.slot >= 0) { - if (options_.max_recursion_depth != 0) { - SetRecursiveStep( - CreateDirectSlotIdentStep(ident_expr.name(), slot.slot, expr.id()), - 1); - } else { AddStep(ExpressionStep::MakeReadSlotStep(slot.slot, expr.id())); - } return; } @@ -900,24 +839,13 @@ class FlatExprVisitor : public cel::AstVisitor { if (const_value) { // If the path starts with a dot, strip it. absl::string_view name = absl::StripPrefix(path, "."); - if (options_.max_recursion_depth != 0) { - SetRecursiveStep( - CreateDirectShadowableValueStep( - name, std::move(const_value).value(), select_root_id), - 1); - return; - } AddStep(CreateShadowableValueStep(name, std::move(const_value).value()), select_root_id); return; } absl::string_view ident_name = absl::StripPrefix(ident_expr.name(), "."); - if (options_.max_recursion_depth != 0) { - SetRecursiveStep(CreateDirectIdentStep(ident_name, expr.id()), 1); - } else { AddStep(ExpressionStep::MakeIdentifierStep(ident_name, expr.id())); - } } void PreVisitSelect(const cel::Expr& expr, @@ -1003,22 +931,6 @@ class FlatExprVisitor : public cel::AstVisitor { } } } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto deps = ExtractRecursiveDependencies(); - if (deps.size() != 1) { - SetProgressStatusIfError(absl::InternalError( - "unexpected number of dependencies for select operation.")); - return; - } - - SetRecursiveStep( - CreateDirectSelectStep(std::move(deps[0]), std::move(field), - select_expr.test_only(), expr.id(), - options_.enable_empty_wrapper_null_unboxing, - enable_optional_types_), - *depth + 1); - return; - } if (field_type.has_value()) { AddStep( @@ -1124,196 +1036,6 @@ class FlatExprVisitor : public cel::AstVisitor { } } - // Returns the maximum recursion depth of the current program if it is - // eligible for recursion, or nullopt if it is not. - std::optional RecursionEligible() { - if (!PlanRecursiveProgram() || program_builder_.current() == nullptr) { - return std::nullopt; - } - return program_builder_.current()->RecursiveDependencyDepth(); - } - - std::vector> - ExtractRecursiveDependencies() { - // Must check recursion eligibility before calling. - ABSL_DCHECK(program_builder_.current() != nullptr); - - return program_builder_.current()->ExtractRecursiveDependencies(); - } - - void MakeTernaryRecursive(const cel::Expr* expr) { - if (expr->call_expr().args().size() != 3) { - SetProgressStatusIfError(absl::InvalidArgumentError( - "unexpected number of args for builtin ternary")); - return; - } - - const cel::Expr* condition_expr = &expr->call_expr().args()[0]; - const cel::Expr* left_expr = &expr->call_expr().args()[1]; - const cel::Expr* right_expr = &expr->call_expr().args()[2]; - - auto* condition_plan = program_builder_.GetSubexpression(condition_expr); - auto* left_plan = program_builder_.GetSubexpression(left_expr); - auto* right_plan = program_builder_.GetSubexpression(right_expr); - - if (condition_plan == nullptr || !condition_plan->IsRecursive() || - left_plan == nullptr || !left_plan->IsRecursive() || - right_plan == nullptr || !right_plan->IsRecursive()) { - SetProgressStatusIfError(FailedRecursivePlanning()); - return; - } - - int max_depth = std::max({0, condition_plan->recursive_program().depth, - left_plan->recursive_program().depth, - right_plan->recursive_program().depth}); - - SetRecursiveStep( - CreateDirectTernaryStep(condition_plan->ExtractRecursiveProgram().step, - left_plan->ExtractRecursiveProgram().step, - right_plan->ExtractRecursiveProgram().step, - expr->id(), options_.short_circuiting), - max_depth + 1); - } - - void MakeShortcircuitRecursive(const cel::Expr* expr, bool is_or) { - int args_size = expr->call_expr().args().size(); - if (args_size < 2) { - SetProgressStatusIfError(absl::InvalidArgumentError( - "unexpected number of args for builtin boolean operator &&/||")); - return; - } - - auto* current_plan = - program_builder_.GetSubexpression(&expr->call_expr().args()[0]); - if (current_plan == nullptr || !current_plan->IsRecursive()) { - SetProgressStatusIfError(FailedRecursivePlanning()); - return; - } - int current_depth = current_plan->recursive_program().depth; - std::unique_ptr current_step = - current_plan->ExtractRecursiveProgram().step; - - for (int i = 1; i < args_size; ++i) { - auto* next_plan = - program_builder_.GetSubexpression(&expr->call_expr().args()[i]); - if (next_plan == nullptr || !next_plan->IsRecursive()) { - SetProgressStatusIfError(FailedRecursivePlanning()); - return; - } - current_depth = - std::max(current_depth, next_plan->recursive_program().depth); - std::unique_ptr next_step = - next_plan->ExtractRecursiveProgram().step; - if (is_or) { - current_step = - CreateDirectOrStep(std::move(current_step), std::move(next_step), - expr->id(), options_.short_circuiting); - } else { - current_step = - CreateDirectAndStep(std::move(current_step), std::move(next_step), - expr->id(), options_.short_circuiting); - } - current_depth++; - } - SetRecursiveStep(std::move(current_step), current_depth); - } - - void MakeOptionalShortcircuit(const cel::Expr* expr, bool is_or_value) { - if (!expr->call_expr().has_target() || - expr->call_expr().args().size() != 1) { - SetProgressStatusIfError(absl::InvalidArgumentError( - "unexpected number of args for optional.or{Value}")); - return; - } - const cel::Expr* left_expr = &expr->call_expr().target(); - const cel::Expr* right_expr = &expr->call_expr().args()[0]; - - auto* left_plan = program_builder_.GetSubexpression(left_expr); - auto* right_plan = program_builder_.GetSubexpression(right_expr); - - if (left_plan == nullptr || !left_plan->IsRecursive() || - right_plan == nullptr || !right_plan->IsRecursive()) { - SetProgressStatusIfError(FailedRecursivePlanning()); - return; - } - int max_depth = std::max({0, left_plan->recursive_program().depth, - right_plan->recursive_program().depth}); - - SetRecursiveStep(CreateDirectOptionalOrStep( - expr->id(), left_plan->ExtractRecursiveProgram().step, - right_plan->ExtractRecursiveProgram().step, - is_or_value, options_.short_circuiting), - max_depth + 1); - } - - void MaybeMakeBindRecursive(const cel::Expr* expr, - const cel::ComprehensionExpr* comprehension, - size_t accu_slot) { - if (!PlanRecursiveProgram()) { - return; - } - - auto* result_plan = - program_builder_.GetSubexpression(&comprehension->result()); - - if (result_plan == nullptr || !result_plan->IsRecursive()) { - SetProgressStatusIfError(FailedRecursivePlanning()); - return; - } - - int result_depth = result_plan->recursive_program().depth; - - auto program = result_plan->ExtractRecursiveProgram(); - SetRecursiveStep( - CreateDirectBindStep(accu_slot, std::move(program.step), expr->id()), - result_depth + 1); - } - - void MaybeMakeComprehensionRecursive( - const cel::Expr* expr, const cel::ComprehensionExpr* comprehension, - size_t iter_slot, size_t iter2_slot, size_t accu_slot) { - if (!PlanRecursiveProgram()) { - return; - } - - auto* accu_plan = - program_builder_.GetSubexpression(&comprehension->accu_init()); - auto* range_plan = - program_builder_.GetSubexpression(&comprehension->iter_range()); - auto* loop_plan = - program_builder_.GetSubexpression(&comprehension->loop_step()); - auto* condition_plan = - program_builder_.GetSubexpression(&comprehension->loop_condition()); - auto* result_plan = - program_builder_.GetSubexpression(&comprehension->result()); - if (accu_plan == nullptr || !accu_plan->IsRecursive() || - range_plan == nullptr || !range_plan->IsRecursive() || - loop_plan == nullptr || !loop_plan->IsRecursive() || - condition_plan == nullptr || !condition_plan->IsRecursive() || - result_plan == nullptr || !result_plan->IsRecursive()) { - SetProgressStatusIfError(FailedRecursivePlanning()); - return; - } - - int max_depth = 0; - max_depth = std::max(max_depth, accu_plan->recursive_program().depth); - max_depth = std::max(max_depth, range_plan->recursive_program().depth); - max_depth = std::max(max_depth, loop_plan->recursive_program().depth); - max_depth = std::max(max_depth, condition_plan->recursive_program().depth); - max_depth = std::max(max_depth, result_plan->recursive_program().depth); - - auto step = CreateDirectComprehensionStep( - iter_slot, iter2_slot, accu_slot, - range_plan->ExtractRecursiveProgram().step, - accu_plan->ExtractRecursiveProgram().step, - loop_plan->ExtractRecursiveProgram().step, - condition_plan->ExtractRecursiveProgram().step, - result_plan->ExtractRecursiveProgram().step, options_.short_circuiting, - expr->id()); - - SetRecursiveStep(std::move(step), max_depth + 1); - } - // Invoked after all child nodes are processed. void PostVisitCall(const cel::Expr& expr, const cel::CallExpr& call_expr) override { @@ -1564,10 +1286,6 @@ class FlatExprVisitor : public cel::AstVisitor { comprehension_stack_.back(); if (comprehension.is_optimizable_list_append) { if (&(comprehension.comprehension->accu_init()) == &expr) { - if (PlanRecursiveProgram()) { - SetRecursiveStep(CreateDirectMutableListStep(expr.id()), 1); - return; - } AddStep(ExpressionStep::MakeNewMutableListStep(expr.id())); return; } @@ -1578,18 +1296,6 @@ class FlatExprVisitor : public cel::AstVisitor { } } } - if (std::optional depth = RecursionEligible(); depth.has_value()) { - auto deps = ExtractRecursiveDependencies(); - if (deps.size() != list_expr.elements().size()) { - SetProgressStatusIfError(absl::InternalError( - "Unexpected number of plan elements for CreateList expr")); - return; - } - auto step = CreateDirectListStep( - std::move(deps), MakeOptionalIndicesSet(list_expr), expr.id()); - SetRecursiveStep(std::move(step), *depth + 1); - return; - } AddStep(CreateCreateListStep(list_expr), expr.id()); } @@ -1612,20 +1318,6 @@ class FlatExprVisitor : public cel::AstVisitor { std::vector fields = std::move(status_or_resolved_fields.value().second); - if (auto depth = RecursionEligible(); depth.has_value()) { - auto deps = ExtractRecursiveDependencies(); - if (deps.size() != struct_expr.fields().size()) { - SetProgressStatusIfError(absl::InternalError( - "Unexpected number of plan elements for CreateStruct expr")); - return; - } - auto step = CreateDirectCreateStructStep( - std::move(resolved_name), std::move(fields), std::move(deps), - MakeOptionalIndicesSet(struct_expr), expr.id()); - SetRecursiveStep(std::move(step), *depth + 1); - return; - } - AddStep(CreateCreateStructStep(std::move(resolved_name), std::move(fields), MakeOptionalIndicesSet(struct_expr)), expr.id()); @@ -1643,28 +1335,12 @@ class FlatExprVisitor : public cel::AstVisitor { comprehension_stack_.back(); if (comprehension.is_optimizable_map_insert) { if (&(comprehension.comprehension->accu_init()) == &expr) { - if (PlanRecursiveProgram()) { - SetRecursiveStep(CreateDirectMutableMapStep(expr.id()), 1); - return; - } AddStep(CreateMutableMapStep(), expr.id()); return; } } } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto deps = ExtractRecursiveDependencies(); - if (deps.size() != 2 * map_expr.entries().size()) { - SetProgressStatusIfError(absl::InternalError( - "Unexpected number of plan elements for CreateStruct expr")); - return; - } - auto step = CreateDirectCreateMapStep( - std::move(deps), MakeOptionalIndicesSet(map_expr), expr.id()); - SetRecursiveStep(std::move(step), *depth + 1); - return; - } AddStep(CreateCreateStructStepForMap(map_expr.entries().size(), MakeOptionalIndicesSet(map_expr)), expr.id()); @@ -1692,14 +1368,6 @@ class FlatExprVisitor : public cel::AstVisitor { auto lazy_overloads = resolver_.FindLazyOverloads( function, call_expr->has_target(), num_args, expr->id()); if (!lazy_overloads.empty()) { - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = program_builder_.current()->ExtractRecursiveDependencies(); - SetRecursiveStep(CreateDirectLazyFunctionStep( - expr->id(), *call_expr, std::move(args), - std::move(lazy_overloads)), - *depth + 1); - return; - } AddStep(ExpressionStep::MakeLazyFunctionStep( CreateLazyFunctionStep(*call_expr, expr->id(), std::move(lazy_overloads)), @@ -1725,18 +1393,6 @@ class FlatExprVisitor : public cel::AstVisitor { } } - if (auto recursion_depth = RecursionEligible(); - recursion_depth.has_value()) { - // Nonnull while active -- nullptr indicates logic error elsewhere in the - // builder. - ABSL_DCHECK(program_builder_.current() != nullptr); - auto args = program_builder_.current()->ExtractRecursiveDependencies(); - SetRecursiveStep( - CreateDirectFunctionStep(expr->id(), *call_expr, std::move(args), - std::move(overloads)), - *recursion_depth + 1); - return; - } AddStep(ExpressionStep::MakeEagerFunctionStep( CreateFunctionStep(*call_expr, expr->id(), std::move(overloads)), expr->id())); @@ -1778,23 +1434,6 @@ class FlatExprVisitor : public cel::AstVisitor { return nullptr; } - void SetRecursiveStep(std::unique_ptr step, int depth) { - if (!progress_status_.ok() || PlanningSuppressed()) { - return; - } - if (program_builder_.current() == nullptr) { - SetProgressStatusIfError(absl::InternalError( - "CEL AST traversal out of order in flat_expr_builder.")); - return; - } - program_builder_.current()->set_recursive_program(std::move(step), depth); - if (depth > max_recursion_depth_) { - SetProgressStatusIfError(absl::InvalidArgumentError( - absl::StrCat("Maximum recursion depth of ", - options_.max_recursion_depth, " exceeded"))); - } - } - void SetProgressStatusIfError(const absl::Status& status) { if (progress_status_.ok() && !status.ok()) { progress_status_ = status; @@ -2005,7 +1644,6 @@ class FlatExprVisitor : public cel::AstVisitor { bool enable_optional_types_; std::optional block_; - int max_recursion_depth_ = 0; }; FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleIndex( @@ -2020,19 +1658,6 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleIndex( return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 2) { - SetProgressStatusIfError(absl::InvalidArgumentError( - "unexpected number of args for builtin index operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep( - CreateDirectContainerAccessStep(std::move(args[0]), std::move(args[1]), - enable_optional_types_, expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } AddStep(CreateContainerAccessStep(call_expr, enable_optional_types_), expr.id()); return CallHandlerResult::kIntercepted; @@ -2047,17 +1672,6 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNot( return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 1) { - SetProgressStatusIfError(absl::InvalidArgumentError( - "unexpected number of args for builtin not operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep(CreateDirectNotStep(std::move(args[0]), expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } AddStep(ExpressionStep::MakeBooleanNotStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -2070,19 +1684,6 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleNotStrictlyFalse( return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 1) { - SetProgressStatusIfError( - absl::InvalidArgumentError("unexpected number of args for builtin " - "@not_strictly_false operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep( - CreateDirectNotStrictlyFalseStep(std::move(args[0]), expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } AddStep(ExpressionStep::MakeNotStrictlyFalseStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -2101,28 +1702,6 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleBlock( block.in = false; index_manager().ReleaseSlots(block.slot_count); - // Check if eligible for recursion and update the plan if so. - // - // The first argument to @block is the list of initializers. These don't - // generate a plan in the main program (they are tracked separately to support - // lazy evaluation) so we only need to extract the second argument -- the body - // of the block that uses the initializers. - ProgramBuilder::Subexpression* body_subexpression = - program_builder_.GetSubexpression(&call_expr.args()[1]); - - if (options_.max_recursion_depth != 0 && body_subexpression != nullptr && - body_subexpression->IsRecursive() && - (options_.max_recursion_depth < 0 || - body_subexpression->recursive_program().depth < - options_.max_recursion_depth)) { - auto recursive_program = body_subexpression->ExtractRecursiveProgram(); - SetRecursiveStep( - CreateDirectBlockStep(block.index, block.slot_count, - std::move(recursive_program.step), expr.id()), - recursive_program.depth + 1); - return CallHandlerResult::kIntercepted; - } - // Otherwise, iterative plan. if (block.slot_count > 0) { AddStep(ExpressionStep::MakeClearSlotsStep(block.index, block.slot_count, @@ -2158,12 +1737,7 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleListAppend( loop_step.call_expr().args().size() == 3 && &(loop_step.call_expr().args()[1]) == &expr; if (is_map_loop_step || is_filter_loop_step) { - if (RecursionEligible().has_value()) { - AddResolvedFunctionStep(&call_expr, &expr, - cel::builtin::kRuntimeListAppend); - } else { AddStep(ExpressionStep::MakeMutableListAppendStep(expr.id())); - } return CallHandlerResult::kIntercepted; } } @@ -2179,19 +1753,6 @@ FlatExprVisitor::CallHandlerResult FlatExprVisitor::HandleHeterogeneousEquality( return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 2) { - SetProgressStatusIfError(absl::InvalidArgumentError( - "unexpected number of args for builtin equality operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep( - CreateDirectEqualityStep(std::move(args[0]), std::move(args[1]), - inequality, expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } AddStep(inequality ? ExpressionStep::MakeFastNotEqualStep(expr.id()) : ExpressionStep::MakeFastEqualStep(expr.id())); return CallHandlerResult::kIntercepted; @@ -2205,19 +1766,6 @@ FlatExprVisitor::HandleHeterogeneousEqualityIn(const cel::Expr& expr, return CallHandlerResult::kIntercepted; } - if (auto depth = RecursionEligible(); depth.has_value()) { - auto args = ExtractRecursiveDependencies(); - if (args.size() != 2) { - SetProgressStatusIfError(absl::InvalidArgumentError( - "unexpected number of args for builtin 'in' operator")); - return CallHandlerResult::kIntercepted; - } - SetRecursiveStep( - CreateDirectInStep(std::move(args[0]), std::move(args[1]), expr.id()), - *depth + 1); - return CallHandlerResult::kIntercepted; - } - AddStep(ExpressionStep::MakeFastInStep(expr.id())); return CallHandlerResult::kIntercepted; } @@ -2246,9 +1794,6 @@ void LogicalCondVisitor::PreVisit(const cel::Expr* expr) { } void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return; - } const int last_arg_index = expr->call_expr().args().size() - 1; const size_t num_args = expr->call_expr().args().size(); if (arg_num == last_arg_index) { @@ -2288,9 +1833,6 @@ void LogicalCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { } void LogicalCondVisitor::PostVisit(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - visitor_->MakeShortcircuitRecursive(expr, is_or_); - } } void OptionalOrCondVisitor::PreVisit(const cel::Expr* expr) { @@ -2300,9 +1842,6 @@ void OptionalOrCondVisitor::PreVisit(const cel::Expr* expr) { } void OptionalOrCondVisitor::PostVisitTarget(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return; - } if (!short_circuiting_) { return; } @@ -2314,11 +1853,6 @@ void OptionalOrCondVisitor::PostVisitTarget(const cel::Expr* expr) { } void OptionalOrCondVisitor::PostVisit(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - visitor_->MakeOptionalShortcircuit(expr, is_or_value_); - return; - } - visitor_->AddStep(CreateOptionalOrStep(is_or_value_), expr->id()); if (!short_circuiting_) { return; @@ -2343,9 +1877,6 @@ void TernaryCondVisitor::PreVisit(const cel::Expr* expr) { } void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return; - } // Ternary operator "_?_:_" requires a special handing. // In contrary to regular function call, its execution affects the control // flow of the overall CEL expression. @@ -2395,10 +1926,6 @@ void TernaryCondVisitor::PostVisitArg(int arg_num, const cel::Expr* expr) { } void TernaryCondVisitor::PostVisit(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - visitor_->MakeTernaryRecursive(expr); - return; - } // Determine and set jump offset in jump instruction. ExpressionStep* cond_step = cond_jump_pos_.has_value() @@ -2445,10 +1972,6 @@ void ExhaustiveTernaryCondVisitor::PreVisit(const cel::Expr* expr) { } void ExhaustiveTernaryCondVisitor::PostVisit(const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - visitor_->MakeTernaryRecursive(expr); - return; - } visitor_->AddStep(CreateTernaryStep(), expr->id()); } @@ -2486,9 +2009,6 @@ ComprehensionNextStep* absl_nullable ComprehensionVisitor::GetNextStep() { absl::Status ComprehensionVisitor::PostVisitArgDefault( cel::ComprehensionArg arg_num, const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return absl::OkStatus(); - } switch (arg_num) { case cel::ITER_RANGE: { init_step_pos_ = visitor_->GetCurrentIndex(); @@ -2592,9 +2112,6 @@ absl::Status ComprehensionVisitor::PostVisitArgDefault( void ComprehensionVisitor::PostVisitArgTrivial(cel::ComprehensionArg arg_num, const cel::Expr* expr) { - if (visitor_->PlanRecursiveProgram()) { - return; - } switch (arg_num) { case cel::ITER_RANGE: { break; @@ -2620,13 +2137,6 @@ void ComprehensionVisitor::PostVisitArgTrivial(cel::ComprehensionArg arg_num, } void ComprehensionVisitor::PostVisit(const cel::Expr* expr) { - if (is_trivial_) { - visitor_->MaybeMakeBindRecursive(expr, &expr->comprehension_expr(), - accu_slot_); - return; - } - visitor_->MaybeMakeComprehensionRecursive( - expr, &expr->comprehension_expr(), iter_slot_, iter2_slot_, accu_slot_); } // Flattens the expression table into the end of the mainline expression vector @@ -2726,13 +2236,6 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( program_builder, extension_context, enable_optional_types_); - if (options_.max_recursion_depth == -1 || options_.max_recursion_depth > 0) { - int depth_limit = options_.max_recursion_depth == -1 - ? std::numeric_limits::max() - : options_.max_recursion_depth; - visitor.SetMaxRecursionDepth(depth_limit); - } - cel::TraversalOptions opts; opts.use_comprehension_callbacks = true; AstTraverse(ast->root_expr(), visitor, opts); diff --git a/eval/compiler/flat_expr_builder_comprehensions_test.cc b/eval/compiler/flat_expr_builder_comprehensions_test.cc index ecea6058c..ed0f67912 100644 --- a/eval/compiler/flat_expr_builder_comprehensions_test.cc +++ b/eval/compiler/flat_expr_builder_comprehensions_test.cc @@ -50,24 +50,18 @@ using ::cel::expr::CheckedExpr; using ::cel::expr::ParsedExpr; using ::testing::HasSubstr; -class CelExpressionBuilderFlatImplComprehensionsTest - : public testing::TestWithParam { +class CelExpressionBuilderFlatImplComprehensionsTest : public testing::Test { public: CelExpressionBuilderFlatImplComprehensionsTest() = default; - bool enable_recursive_planning() { return GetParam(); } - cel::RuntimeOptions GetRuntimeOptions() { cel::RuntimeOptions options; - if (enable_recursive_planning()) { - options.max_recursion_depth = -1; - } options.enable_comprehension_list_append = true; return options; } }; -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, NestedComp) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, NestedComp) { cel::RuntimeOptions options = GetRuntimeOptions(); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -85,7 +79,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, NestedComp) { EXPECT_THAT(*result.ListOrDie(), testing::SizeIs(2)); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, MapComp) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, MapComp) { cel::RuntimeOptions options = GetRuntimeOptions(); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -106,7 +100,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, MapComp) { test::EqualsCelValue(CelValue::CreateInt64(4))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneTrue) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneTrue) { cel::RuntimeOptions options = GetRuntimeOptions(); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -123,7 +117,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneTrue) { EXPECT_THAT(result, test::IsCelBool(true)); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneFalse) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneFalse) { cel::RuntimeOptions options = GetRuntimeOptions(); CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -140,7 +134,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ExistsOneFalse) { EXPECT_THAT(result, test::IsCelBool(false)); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ListCompWithUnknowns) { +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ListCompWithUnknowns) { cel::RuntimeOptions options = GetRuntimeOptions(); options.unknown_processing = UnknownProcessingOptions::kAttributeAndFunction; CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); @@ -179,7 +173,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, ListCompWithUnknowns) { testing::Eq(1)); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, InvalidComprehensionWithRewrite) { CheckedExpr expr; // The rewrite step which occurs when an identifier gets a more qualified name @@ -216,7 +210,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("Invalid empty expression")))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithConcatVulernability) { CheckedExpr expr; // The comprehension loop step performs an unsafe concatenation of the @@ -270,7 +264,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithListVulernability) { CheckedExpr expr; // The comprehension @@ -314,7 +308,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithStructVulernability) { CheckedExpr expr; // The comprehension loop step builds a deeply nested struct which expands @@ -371,7 +365,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithNestedComprehensionResultVulernability) { CheckedExpr expr; // The nested comprehension performs an unsafe concatenation on the parent @@ -439,7 +433,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithNestedComprehensionLoopStepVulernability) { CheckedExpr expr; // The nested comprehension performs an unsafe concatenation on the parent @@ -486,7 +480,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithNestedComprehensionLoopStepVulernabilityResult) { CheckedExpr expr; // The nested comprehension performs an unsafe concatenation on the parent @@ -538,7 +532,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, ComprehensionWithNestedComprehensionLoopStepIterRangeVulnerability) { CheckedExpr expr; // The nested comprehension unsafely modifies the parent accumulator @@ -585,7 +579,7 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("memory exhaustion vulnerability"))); } -TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, +TEST_F(CelExpressionBuilderFlatImplComprehensionsTest, InvalidBindComprehension) { ParsedExpr expr; // Trivial comprehensions (such as cel.bind), are optimized by skipping the @@ -629,13 +623,6 @@ TEST_P(CelExpressionBuilderFlatImplComprehensionsTest, HasSubstr("Unexpected iter_var access in trivial comprehension"))); } -INSTANTIATE_TEST_SUITE_P(TestSuite, - CelExpressionBuilderFlatImplComprehensionsTest, - testing::Bool(), - [](const testing::TestParamInfo& info) { - return info.param ? "recursive" : "default"; - }); - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/compiler/flat_expr_builder_extensions.cc b/eval/compiler/flat_expr_builder_extensions.cc index 7e1417eb7..3f308125a 100644 --- a/eval/compiler/flat_expr_builder_extensions.cc +++ b/eval/compiler/flat_expr_builder_extensions.cc @@ -13,7 +13,6 @@ // limitations under the License. #include "eval/compiler/flat_expr_builder_extensions.h" -#include #include #include #include @@ -27,10 +26,8 @@ #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "absl/types/variant.h" #include "common/expr.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" namespace google::api::expr::runtime { @@ -39,31 +36,6 @@ namespace { using Subexpression = google::api::expr::runtime::ProgramBuilder::Subexpression; -// Remap a recursive program to its parent if the parent is a transparent -// wrapper. -void MaybeReassignChildRecursiveProgram(Subexpression* parent) { - if (parent->IsFlattened() || parent->IsRecursive()) { - return; - } - if (parent->elements().size() != 1) { - return; - } - auto* child_alternative = - absl::get_if(&parent->elements()[0]); - if (child_alternative == nullptr) { - return; - } - - auto& child_subexpression = *child_alternative; - if (!child_subexpression->IsRecursive()) { - return; - } - - auto child_program = child_subexpression->ExtractRecursiveProgram(); - parent->set_recursive_program(std::move(child_program.step), - child_program.depth); -} - } // namespace Subexpression::Subexpression(const cel::Expr* self, ProgramBuilder* owner) @@ -72,8 +44,6 @@ Subexpression::Subexpression(const cel::Expr* self, ProgramBuilder* owner) size_t Subexpression::ComputeSize() const { if (IsFlattened()) { return flattened_elements().size(); - } else if (IsRecursive()) { - return 1; } std::vector to_expand{this}; size_t size = 0; @@ -83,9 +53,6 @@ size_t Subexpression::ComputeSize() const { if (expr->IsFlattened()) { size += expr->flattened_elements().size(); continue; - } else if (expr->IsRecursive()) { - size += 1; - continue; } for (const auto& elem : expr->elements()) { if (auto* child = absl::get_if(&elem); child != nullptr) { @@ -98,45 +65,6 @@ size_t Subexpression::ComputeSize() const { return size; } -std::optional Subexpression::RecursiveDependencyDepth() const { - auto* tree = absl::get_if(&program_); - int depth = 0; - if (tree == nullptr) { - return std::nullopt; - } - for (const auto& element : *tree) { - auto* subexpression = absl::get_if(&element); - if (subexpression == nullptr) { - return std::nullopt; - } - if (!(*subexpression)->IsRecursive()) { - return std::nullopt; - } - depth = std::max(depth, (*subexpression)->recursive_program().depth); - } - return depth; -} - -std::vector> -Subexpression::ExtractRecursiveDependencies() const { - auto* tree = absl::get_if(&program_); - std::vector> dependencies; - if (tree == nullptr) { - return {}; - } - for (const auto& element : *tree) { - auto* subexpression = absl::get_if(&element); - if (subexpression == nullptr) { - return {}; - } - if (!(*subexpression)->IsRecursive()) { - return {}; - } - dependencies.push_back((*subexpression)->ExtractRecursiveProgram().step); - } - return dependencies; -} - Subexpression* absl_nullable Subexpression::ExtractChild(Subexpression* child) { ABSL_DCHECK(child != nullptr); if (IsFlattened()) { @@ -161,7 +89,6 @@ Subexpression* absl_nullable Subexpression::ExtractChild(Subexpression* child) { // target step. int Subexpression::CalculateOffset(int base, int target) const { ABSL_DCHECK(!IsFlattened()); - ABSL_DCHECK(!IsRecursive()); int sign = 1; int start = base + 1; @@ -220,13 +147,6 @@ void Subexpression::Flatten() { absl::c_move(elements, std::back_inserter(flat)); elements.clear(); continue; - } else if (subexpr->IsRecursive()) { - flat.push_back(ExpressionStep::MakeGenericStep( - std::make_unique( - std::move(subexpr->ExtractRecursiveProgram().step), - subexpr->self_->id()), - subexpr->self_->id())); - continue; } auto& elements = subexpr->elements(); size_t size = elements.size(); @@ -253,13 +173,6 @@ void Subexpression::Flatten() { program_ = std::move(flat); } -Subexpression::RecursiveProgram Subexpression::ExtractRecursiveProgram() { - ABSL_DCHECK(IsRecursive()); - auto result = std::move(absl::get(program_)); - program_.emplace>(); - return result; -} - bool Subexpression::ExtractTo(ExecutionPath& out) { if (!IsFlattened()) { return false; @@ -328,8 +241,6 @@ Subexpression* absl_nullable ProgramBuilder::ExitSubexpression( ABSL_DCHECK(expr == current_->self_); ABSL_DCHECK(GetSubexpression(expr) == current_); - MaybeReassignChildRecursiveProgram(current_); - Subexpression* result = GetSubexpression(current_->parent_); ABSL_DCHECK(result != nullptr || current_ == root_); current_ = result; @@ -350,9 +261,6 @@ ExpressionStep* absl_nullable ProgramBuilder::AddStep(ExpressionStep step) { if (current_ == nullptr) { return nullptr; } - if (current_->IsRecursive()) { - return nullptr; - } if (current_->IsFlattened()) { current_->flattened_elements().push_back(std::move(step)); return ¤t_->flattened_elements().back(); @@ -449,19 +357,6 @@ void ProgramBuilder::Reset() { subprogram_map_.clear(); } -absl::Status PlannerContext::ReplaceSubplan( - const cel::Expr& node, std::unique_ptr step, - int depth) { - auto* subexpression = program_builder_.GetSubexpression(&node); - if (subexpression == nullptr) { - return absl::InternalError( - "attempted to update program step for untracked expr node"); - } - - subexpression->set_recursive_program(std::move(step), depth); - return absl::OkStatus(); -} - absl::Status PlannerContext::AddSubplanStep(const cel::Expr& node, ExpressionStep step) { auto* subexpression = program_builder_.GetSubexpression(&node); diff --git a/eval/compiler/flat_expr_builder_extensions.h b/eval/compiler/flat_expr_builder_extensions.h index e85f337f9..66b80d215 100644 --- a/eval/compiler/flat_expr_builder_extensions.h +++ b/eval/compiler/flat_expr_builder_extensions.h @@ -36,19 +36,14 @@ #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/types/optional.h" #include "absl/types/variant.h" #include "base/ast.h" #include "base/type_provider.h" #include "common/expr.h" -#include "common/native_type.h" #include "common/type_reflector.h" #include "eval/compiler/resolver.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_logic.h" -#include "eval/eval/trace_step.h" -#include "internal/casts.h" #include "runtime/internal/issue_collector.h" #include "runtime/internal/runtime_env.h" #include "runtime/runtime_options.h" @@ -93,11 +88,6 @@ class ProgramBuilder { using FlattenedPlan = ExecutionPath; public: - struct RecursiveProgram { - std::unique_ptr step; - int depth; - }; - ~Subexpression() = default; // Not copyable or movable. @@ -108,10 +98,6 @@ class ProgramBuilder { // Add a program step at the current end of the subexpression. bool AddStep(ExpressionStep step) { - if (IsRecursive()) { - return false; - } - if (IsFlattened()) { flattened_elements().push_back(std::move(step)); return true; @@ -160,27 +146,6 @@ class ProgramBuilder { return absl::get(program_); } - void set_recursive_program(std::unique_ptr step, - int depth) { - program_ = RecursiveProgram{std::move(step), depth}; - } - - const RecursiveProgram& recursive_program() const { - ABSL_DCHECK(IsRecursive()); - return absl::get(program_); - } - - absl::optional RecursiveDependencyDepth() const; - - std::vector> - ExtractRecursiveDependencies() const; - - RecursiveProgram ExtractRecursiveProgram(); - - bool IsRecursive() const { - return absl::holds_alternative(program_); - } - // Compute the current number of program steps in this subexpression and // its dependencies. size_t ComputeSize() const; @@ -221,7 +186,7 @@ class ProgramBuilder { // // This adds complexity, but supports swapping to a flat representation as // needed. - absl::variant program_; + absl::variant program_; const cel::Expr* self_; const cel::Expr* absl_nullable parent_; @@ -314,31 +279,6 @@ class ProgramBuilder { SubprogramMap subprogram_map_; }; -// Attempt to downcast a specific type of recursive step. -template -const Subclass* TryDowncastDirectStep(const DirectExpressionStep* step) { - if (step == nullptr) { - return nullptr; - } - - auto type_id = step->GetNativeTypeId(); - if (type_id == cel::NativeTypeId::For()) { - const auto* trace_step = cel::internal::down_cast(step); - auto deps = trace_step->GetDependencies(); - if (!deps.has_value() || deps->size() != 1) { - return nullptr; - } - step = deps->at(0); - type_id = step->GetNativeTypeId(); - } - - if (type_id == cel::NativeTypeId::For()) { - return cel::internal::down_cast(step); - } - - return nullptr; -} - // Class representing FlatExpr internals exposed to extensions. class PlannerContext { public: @@ -387,14 +327,6 @@ class PlannerContext { // expr->program mapping for any descendants. absl::Status ReplaceSubplan(const cel::Expr& node, ExecutionPath path); - // Replace the subplan associated with node with a new recursive subplan. - // - // This operation clears any existing plan to which removes the - // expr->program mapping for any descendants. - absl::Status ReplaceSubplan(const cel::Expr& node, - std::unique_ptr step, - int depth); - // Extend the current subplan with the given expression step. absl::Status AddSubplanStep(const cel::Expr& node, ExpressionStep step); absl::Status AddSubplanStep(const cel::Expr& node, diff --git a/eval/compiler/flat_expr_builder_extensions_test.cc b/eval/compiler/flat_expr_builder_extensions_test.cc index 2eae196af..07a6cfbe6 100644 --- a/eval/compiler/flat_expr_builder_extensions_test.cc +++ b/eval/compiler/flat_expr_builder_extensions_test.cc @@ -15,21 +15,16 @@ #include #include -#include #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "common/expr.h" -#include "common/native_type.h" -#include "common/value.h" #include "eval/compiler/resolver.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "eval/eval/function_step.h" -#include "internal/status_macros.h" #include "internal/testing.h" #include "runtime/function_registry.h" #include "runtime/internal/issue_collector.h" @@ -53,7 +48,6 @@ using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::testing::ElementsAre; using ::testing::IsEmpty; -using ::testing::Optional; class TestStepLogic : public ExpressionStepLogic { public: @@ -526,55 +520,5 @@ TEST_F(ProgramBuilderTest, ExtractToRequiresFlatten) { UniquePtrHolds(step_ptrs.a))); } -TEST_F(ProgramBuilderTest, Recursive) { - Expr a; - Expr b; - Expr c; - - ProgramBuilder program_builder; - - program_builder.EnterSubexpression(&a); - program_builder.EnterSubexpression(&b); - program_builder.current()->set_recursive_program( - CreateConstValueDirectStep(cel::NullValue()), 1); - program_builder.ExitSubexpression(&b); - program_builder.EnterSubexpression(&c); - program_builder.current()->set_recursive_program( - CreateConstValueDirectStep(cel::NullValue()), 1); - program_builder.ExitSubexpression(&c); - - ASSERT_FALSE(program_builder.current()->IsFlattened()); - ASSERT_FALSE(program_builder.current()->IsRecursive()); - ASSERT_TRUE(program_builder.GetSubexpression(&b)->IsRecursive()); - ASSERT_TRUE(program_builder.GetSubexpression(&c)->IsRecursive()); - - EXPECT_EQ(program_builder.GetSubexpression(&b)->recursive_program().depth, 1); - EXPECT_EQ(program_builder.GetSubexpression(&c)->recursive_program().depth, 1); - - cel::CallExpr call_expr; - call_expr.set_function("_==_"); - call_expr.mutable_args().emplace_back(); - call_expr.mutable_args().emplace_back(); - - auto max_depth = program_builder.current()->RecursiveDependencyDepth(); - - EXPECT_THAT(max_depth, Optional(1)); - - auto deps = program_builder.current()->ExtractRecursiveDependencies(); - - program_builder.current()->set_recursive_program( - CreateDirectFunctionStep(-1, call_expr, std::move(deps), {}), - *max_depth + 1); - - program_builder.ExitSubexpression(&a); - - auto path = program_builder.FlattenMain(); - - ASSERT_THAT(path, testing::SizeIs(1)); - EXPECT_TRUE(path[0].IsGenericStep() && - path[0].GetGenericStep()->GetNativeTypeId() == - cel::NativeTypeId::For()); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/compiler/flat_expr_builder_test.cc b/eval/compiler/flat_expr_builder_test.cc index f7cde337b..8611de388 100644 --- a/eval/compiler/flat_expr_builder_test.cc +++ b/eval/compiler/flat_expr_builder_test.cc @@ -25,7 +25,6 @@ #include "cel/expr/syntax.pb.h" #include "google/protobuf/field_mask.pb.h" #include "google/protobuf/descriptor.pb.h" -#include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" @@ -45,7 +44,6 @@ #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_attribute.h" -#include "eval/public/cel_builtins.h" #include "eval/public/cel_expr_builder_factory.h" #include "eval/public/cel_expression.h" #include "eval/public/cel_function.h" @@ -256,7 +254,7 @@ TEST(FlatExprBuilderTest, BinaryCallTooManyArguments) { CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv()); auto* call = expr.mutable_call_expr(); - call->set_function(builtin::kAnd); + call->set_function(cel::builtin::kAnd); call->mutable_target()->mutable_const_expr()->set_string_value("random"); call->add_args()->mutable_const_expr()->set_bool_value(false); call->add_args()->mutable_const_expr()->set_bool_value(true); @@ -270,7 +268,7 @@ TEST(FlatExprBuilderTest, TernaryCallTooManyArguments) { Expr expr; SourceInfo source_info; auto* call = expr.mutable_call_expr(); - call->set_function(builtin::kTernary); + call->set_function(cel::builtin::kTernary); call->mutable_target()->mutable_const_expr()->set_string_value("random"); call->add_args()->mutable_const_expr()->set_bool_value(false); call->add_args()->mutable_const_expr()->set_int64_value(1); @@ -1813,7 +1811,7 @@ absl::Status RunTernaryExpression(CelValue selector, CelValue value1, Expr expr; SourceInfo source_info; auto call_expr = expr.mutable_call_expr(); - call_expr->set_function(builtin::kTernary); + call_expr->set_function(cel::builtin::kTernary); auto arg0 = call_expr->add_args(); arg0->mutable_ident_expr()->set_name("selector"); @@ -1842,7 +1840,7 @@ TEST(FlatExprBuilderTest, Ternary) { Expr expr; SourceInfo source_info; auto call_expr = expr.mutable_call_expr(); - call_expr->set_function(builtin::kTernary); + call_expr->set_function(cel::builtin::kTernary); auto arg0 = call_expr->add_args(); arg0->mutable_ident_expr()->set_name("selector"); @@ -3018,138 +3016,6 @@ INSTANTIATE_TEST_SUITE_P( VariadicLogicalEvalTestCase{"All_Unknown", "[a, b, c].all(x, x)", "true", "unknown1", "true", "unknown"})); -struct RecursionDepthTestCase { - std::string label; - std::string expr; - int max_recursion_depth; - absl::StatusCode expected_status_code; - std::string expected_error_msg; -}; - -class FlatExprBuilderRecursionDepthTest - : public testing::TestWithParam {}; - -TEST_P(FlatExprBuilderRecursionDepthTest, CheckRecursionLimit) { - const auto& test_case = GetParam(); - ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(test_case.expr)); - - cel::RuntimeOptions options; - options.max_recursion_depth = test_case.max_recursion_depth; - options.fail_on_warnings = false; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - - auto result = - builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()); - if (test_case.expected_status_code == absl::StatusCode::kOk) { - EXPECT_THAT(result, IsOk()); - } else { - EXPECT_THAT(result, StatusIs(test_case.expected_status_code, - HasSubstr(test_case.expected_error_msg))); - } -} - -INSTANTIATE_TEST_SUITE_P( - FlatExprBuilderRecursionDepthTest, FlatExprBuilderRecursionDepthTest, - testing::Values( - RecursionDepthTestCase{"AndChildLimitExceeded", "(1 + 1) && true", 1, - absl::StatusCode::kInvalidArgument, - "Maximum recursion depth of 1 exceeded"}, - RecursionDepthTestCase{"AndParentLimitExceeded", "(1 + 1) && true", 2, - absl::StatusCode::kInvalidArgument, - "Maximum recursion depth of 2 exceeded"}, - RecursionDepthTestCase{"AndLimitSuccess", "(1 + 1) && true", 3, - absl::StatusCode::kOk, ""}, - RecursionDepthTestCase{"AndLimitSuccessGenerous", "(1 + 1) && true", 10, - absl::StatusCode::kOk, ""}, - RecursionDepthTestCase{"AndLimitSuccessUnlimited", "(1 + 1) && true", - -1, absl::StatusCode::kOk, ""}, - RecursionDepthTestCase{"OrChildLimitExceeded", "(1 + 1) || true", 1, - absl::StatusCode::kInvalidArgument, - "Maximum recursion depth of 1 exceeded"}, - RecursionDepthTestCase{"OrParentLimitExceeded", "(1 + 1) || true", 2, - absl::StatusCode::kInvalidArgument, - "Maximum recursion depth of 2 exceeded"}, - RecursionDepthTestCase{"OrLimitSuccess", "(1 + 1) || true", 3, - absl::StatusCode::kOk, ""}, - RecursionDepthTestCase{"OrLimitSuccessGenerous", - "(1 + 1) || false || false || false || false || " - "(true && true && true && true && false)", - 10, absl::StatusCode::kOk, ""}, - RecursionDepthTestCase{"OrLimitSuccessUnlimited", "(1 + 1) || true", -1, - absl::StatusCode::kOk, ""}, - RecursionDepthTestCase{"AndDepthUpdateFromSubsequentArg", - "true && (1 + 1 + 1 + 1)", 4, - absl::StatusCode::kInvalidArgument, - "Maximum recursion depth of 4 exceeded"}, - RecursionDepthTestCase{"OrDepthUpdateFromSubsequentArg", - "true || (1 + 1 + 1 + 1)", 4, - absl::StatusCode::kInvalidArgument, - "Maximum recursion depth of 4 exceeded"})); - -TEST(FlatExprBuilderTest, NonRecursiveChildBlockAndError) { - ParsedExpr parsed_expr; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - R"pb( - expr: { - call_expr: { - function: "_&&_" - args { const_expr: { bool_value: true } } - args { - call_expr: { - function: "cel.@block" - args { - list_expr { elements { const_expr: { int64_value: 1 } } } - } - args { ident_expr: { name: "@index0" } } - } - } - } - } - )pb", - &parsed_expr)); - - cel::RuntimeOptions options; - options.max_recursion_depth = 2; - options.fail_on_warnings = false; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - EXPECT_THAT( - builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()), - StatusIs(absl::StatusCode::kInternal, - HasSubstr("failed to build recursive program"))); -} - -TEST(FlatExprBuilderTest, NonRecursiveChildBlockOrError) { - ParsedExpr parsed_expr; - ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( - R"pb( - expr: { - call_expr: { - function: "_||_" - args { const_expr: { bool_value: true } } - args { - call_expr: { - function: "cel.@block" - args { - list_expr { elements { const_expr: { int64_value: 1 } } } - } - args { ident_expr: { name: "@index0" } } - } - } - } - } - )pb", - &parsed_expr)); - - cel::RuntimeOptions options; - options.max_recursion_depth = 2; - options.fail_on_warnings = false; - CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); - EXPECT_THAT( - builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()), - StatusIs(absl::StatusCode::kInternal, - HasSubstr("failed to build recursive program"))); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/compiler/qualified_reference_resolver.cc b/eval/compiler/qualified_reference_resolver.cc index 158e492be..a46034298 100644 --- a/eval/compiler/qualified_reference_resolver.cc +++ b/eval/compiler/qualified_reference_resolver.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -26,7 +27,6 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "base/ast.h" #include "base/builtins.h" #include "common/ast.h" diff --git a/eval/compiler/regex_precompilation_optimization.cc b/eval/compiler/regex_precompilation_optimization.cc index bd3a841ed..f4275ebd3 100644 --- a/eval/compiler/regex_precompilation_optimization.cc +++ b/eval/compiler/regex_precompilation_optimization.cc @@ -33,8 +33,6 @@ #include "common/native_type.h" #include "common/value.h" #include "eval/compiler/flat_expr_builder_extensions.h" -#include "eval/eval/compiler_constant_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/regex_match_step.h" #include "internal/re2_options.h" @@ -175,17 +173,6 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { return std::nullopt; } std::optional constant; - if (subexpression->IsRecursive()) { - const auto& program = subexpression->recursive_program(); - auto deps = program.step->GetDependencies(); - if (deps.has_value() && deps->size() == 2) { - const auto* re_plan = - TryDowncastDirectStep(deps->at(1)); - if (re_plan != nullptr) { - constant = re_plan->value(); - } - } - } else { // otherwise stack-machine program. ExecutionPathView re_plan = context.GetSubplan(re_expr); if (re_plan.size() == 1) { @@ -194,7 +181,6 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { constant = std::move(val); } } - } if (constant.has_value() && constant->IsString()) { return constant->GetString().ToString(); @@ -208,33 +194,10 @@ class RegexPrecompilationOptimization : public ProgramOptimizer { ProgramBuilder::Subexpression* absl_nonnull subexpression, const Expr& call, const Expr& subject, std::shared_ptr regex_program) { - if (subexpression->IsRecursive()) { - return RewriteRecursivePlan(subexpression, call, subject, - std::move(regex_program)); - } return RewriteStackMachinePlan(context, call, subject, std::move(regex_program)); } - absl::Status RewriteRecursivePlan( - ProgramBuilder::Subexpression* absl_nonnull subexpression, - const Expr& call, const Expr& subject, - std::shared_ptr regex_program) { - auto program = subexpression->ExtractRecursiveProgram(); - auto deps = program.step->ExtractDependencies(); - if (!deps.has_value() || deps->size() != 2) { - // Possibly already const-folded, put the plan back. - subexpression->set_recursive_program(std::move(program.step), - program.depth); - return absl::OkStatus(); - } - subexpression->set_recursive_program( - CreateDirectRegexMatchStep(call.id(), std::move(deps->at(0)), - std::move(regex_program)), - program.depth); - return absl::OkStatus(); - } - absl::Status RewriteStackMachinePlan( PlannerContext& context, const Expr& call, const Expr& subject, std::shared_ptr regex_program) { diff --git a/eval/eval/BUILD b/eval/eval/BUILD index ded2492ca..37b88db94 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -56,7 +56,6 @@ cc_library( ":attribute_trail", ":attribute_utility", ":comprehension_slots", - ":direct_expression_step", ":evaluator_stack", ":expression_step_logic", ":iterator_stack", @@ -66,7 +65,6 @@ cc_library( "//common:expr", "//common:function_descriptor", "//common:kind", - "//common:native_type", "//common:value", "//common:value_kind", "//eval/internal:errors", @@ -85,7 +83,6 @@ cc_library( "@com_google_absl//absl/base", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/container:inlined_vector", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", @@ -107,11 +104,7 @@ cc_library( "cel_expression_flat_impl.h", ], deps = [ - ":attribute_trail", - ":comprehension_slots", - ":direct_expression_step", ":evaluator_core", - "//common:native_type", "//common:value", "//eval/internal:adapter_activation_impl", "//eval/internal:interop", @@ -123,10 +116,8 @@ cc_library( "//runtime/internal:runtime_env", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", ], ) @@ -206,10 +197,7 @@ cc_library( hdrs = [ "expression_step_logic.h", ], - deps = [ - "//common:native_type", - "@com_google_absl//absl/status", - ], + deps = ["//common:native_type"], ) cc_library( @@ -217,23 +205,7 @@ cc_library( hdrs = [ "expression_step_base.h", ], - deps = [ - ":evaluator_core", - ":expression_step_logic", - ], -) - -cc_library( - name = "const_value_step", - hdrs = [ - "const_value_step.h", - ], - deps = [ - ":compiler_constant_step", - ":direct_expression_step", - "//common:value", - "@com_google_absl//absl/status:statusor", - ], + deps = [":expression_step_logic"], ) cc_library( @@ -247,9 +219,9 @@ cc_library( deps = [ ":attribute_trail", ":attribute_utility", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//base:attributes", "//common:expr", "//common:kind", @@ -257,7 +229,6 @@ cc_library( "//common:value_kind", "//eval/internal:errors", "//internal:number", - "//internal:status_macros", "//runtime/internal:errors", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", @@ -273,12 +244,10 @@ cc_library( srcs = ["regex_match_step.cc"], hdrs = ["regex_match_step.h"], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:value", - "//internal:status_macros", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:cord", @@ -311,9 +280,9 @@ cc_library( ], deps = [ ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:memory", "//common:type", "//common:value", @@ -342,11 +311,10 @@ cc_library( "create_list_step.h", ], deps = [ - ":attribute_trail", ":attribute_utility", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:expr", "//common:value", "//internal:status_macros", @@ -366,10 +334,9 @@ cc_library( "create_struct_step.h", ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:value", "//internal:status_macros", "@com_google_absl//absl/container:flat_hash_set", @@ -389,10 +356,9 @@ cc_library( "create_map_step.h", ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:value", "//internal:status_macros", "@com_google_absl//absl/container:flat_hash_set", @@ -417,8 +383,8 @@ cc_test( ], deps = [ ":attribute_trail", - ":direct_expression_step", ":evaluator_core", + ":expression_step_logic", "//base:attributes", "//common:value", "//common:value_kind", @@ -444,31 +410,19 @@ cc_test( "comprehension_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":comprehension_slots", - ":const_value_step", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", "//base:data", "//common:expr", - "//common:value", - "//common:value_testing", "//eval/public:activation", "//eval/public:cel_attribute", "//eval/public:cel_value", "//eval/public/structs:cel_proto_wrapper", - "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -514,7 +468,6 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":container_access_step", - ":direct_expression_step", ":evaluator_core", "//base:builtins", "//base:data", @@ -533,6 +486,7 @@ cc_test( "//eval/public/testing:matchers", "//internal:testing", "//parser", + "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/base:nullability", @@ -559,7 +513,6 @@ cc_test( "//internal:testing", "//parser", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_protobuf//:protobuf", @@ -573,22 +526,16 @@ cc_test( "ident_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", ":evaluator_core", "//base:data", "//common:memory", - "//common:value", "//eval/public:activation", "//eval/public:cel_attribute", "//eval/public:cel_value", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/status", "@com_google_protobuf//:protobuf", ], @@ -602,10 +549,7 @@ cc_test( ], deps = [ ":cel_expression_flat_impl", - ":const_value_step", - ":direct_expression_step", ":evaluator_core", - "//base:builtins", "//base:data", "//common:constant", "//common:expr", @@ -623,13 +567,9 @@ cc_test( "//eval/public/testing:matchers", "//eval/testutil:test_message_cc_proto", "//internal:testing", - "//runtime:function_overload_reference", - "//runtime:function_registry", "//runtime:runtime_options", - "//runtime:standard_functions", "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:status_macros", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -645,16 +585,11 @@ cc_test( "logic_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":const_value_step", - ":direct_expression_step", ":evaluator_core", "//base:attributes", "//base:data", "//common:expr", - "//common:unknown", - "//common:value", "//eval/public:activation", "//eval/public:cel_attribute", "//eval/public:cel_value", @@ -662,17 +597,12 @@ cc_test( "//eval/public:unknown_set", "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", "@com_google_protobuf//:protobuf", ], @@ -685,9 +615,7 @@ cc_test( "select_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":const_value_step", ":evaluator_core", ":select_step", "//base:attributes", @@ -696,7 +624,6 @@ cc_test( "//common:legacy_value", "//common:type", "//common:value", - "//common:value_testing", "//eval/public:activation", "//eval/public:cel_attribute", "//eval/public:cel_value", @@ -707,19 +634,13 @@ cc_test( "//eval/public/structs:trivial_legacy_type_info", "//eval/testutil:test_extensions_cc_proto", "//eval/testutil:test_message_cc_proto", - "//extensions/protobuf:value", "//internal:proto_matchers", "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", @@ -739,17 +660,13 @@ cc_test( "create_list_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":const_value_step", ":create_list_step", - ":direct_expression_step", ":evaluator_core", "//base:attributes", "//base:data", "//common:expr", "//common:value", - "//common:value_testing", "//eval/internal:interop", "//eval/public:activation", "//eval/public:cel_attribute", @@ -758,13 +675,9 @@ cc_test( "//eval/public/testing:matchers", "//internal:status_macros", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", @@ -783,11 +696,8 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":create_struct_step", - ":direct_expression_step", ":evaluator_core", - ":ident_step", "//base:data", - "//common:expr", "//eval/public:activation", "//eval/public:cel_type_registry", "//eval/public:cel_value", @@ -822,9 +732,7 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":create_map_step", - ":direct_expression_step", ":evaluator_core", - ":ident_step", "//base:data", "//common:expr", "//eval/public:activation", @@ -928,16 +836,13 @@ cc_library( "ternary_step.h", ], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//base:builtins", "//common:value", "//eval/internal:errors", - "//internal:status_macros", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", ], ) @@ -948,28 +853,21 @@ cc_test( "ternary_step_test.cc", ], deps = [ - ":attribute_trail", ":cel_expression_flat_impl", - ":const_value_step", - ":direct_expression_step", ":evaluator_core", ":ternary_step", "//base:attributes", "//base:data", "//common:expr", - "//common:value", "//eval/public:activation", + "//eval/public:cel_attribute", "//eval/public:cel_value", "//eval/public:unknown_attribute_set", "//eval/public:unknown_set", "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", "//runtime:runtime_options", "//runtime/internal:runtime_env", "//runtime/internal:runtime_env_testing", - "//runtime/internal:runtime_type_provider", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status", "@com_google_protobuf//:protobuf", @@ -981,13 +879,10 @@ cc_library( srcs = ["shadowable_value_step.cc"], hdrs = ["shadowable_value_step.h"], deps = [ - ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", + ":expression_step_logic", "//common:value", - "//internal:status_macros", - "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:string_view", @@ -1014,39 +909,7 @@ cc_test( "//runtime/internal:runtime_env_testing", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/status:statusor", - ], -) - -cc_library( - name = "compiler_constant_step", - srcs = ["compiler_constant_step.cc"], - hdrs = ["compiler_constant_step.h"], - deps = [ - ":attribute_trail", - ":direct_expression_step", - ":evaluator_core", - "//common:native_type", - "//common:value", - "@com_google_absl//absl/status", - ], -) - -cc_test( - name = "compiler_constant_step_test", - srcs = ["compiler_constant_step_test.cc"], - deps = [ - ":attribute_trail", - ":compiler_constant_step", - ":direct_expression_step", - ":evaluator_core", - "//common:native_type", - "//common:value", - "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", - "//runtime:runtime_options", - "//runtime/internal:runtime_type_provider", + "@com_google_absl//absl/strings:string_view", "@com_google_protobuf//:protobuf", ], ) @@ -1056,7 +919,6 @@ cc_test( srcs = ["lazy_init_step_test.cc"], deps = [ ":comprehension_slots", - ":const_value_step", ":evaluator_core", "//base:data", "//common:value", @@ -1070,46 +932,16 @@ cc_test( ], ) -cc_library( - name = "direct_expression_step", - srcs = ["direct_expression_step.cc"], - hdrs = ["direct_expression_step.h"], - deps = [ - ":attribute_trail", - "//common:native_type", - "//common:value", - "@com_google_absl//absl/status", - "@com_google_absl//absl/types:optional", - ], -) - -cc_library( - name = "trace_step", - hdrs = ["trace_step.h"], - deps = [ - ":attribute_trail", - ":direct_expression_step", - ":evaluator_core", - "//common:native_type", - "//common:value", - "//internal:status_macros", - "@com_google_absl//absl/status", - "@com_google_absl//absl/types:optional", - ], -) - cc_library( name = "optional_or_step", srcs = ["optional_or_step.cc"], hdrs = ["optional_or_step.h"], deps = [ ":attribute_trail", - ":direct_expression_step", ":evaluator_core", ":expression_step_base", ":expression_step_logic", "//common:value", - "//internal:status_macros", "//runtime/internal:errors", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status", @@ -1118,31 +950,6 @@ cc_library( ], ) -cc_test( - name = "optional_or_step_test", - srcs = ["optional_or_step_test.cc"], - deps = [ - ":attribute_trail", - ":const_value_step", - ":direct_expression_step", - ":evaluator_core", - ":optional_or_step", - "//common:value", - "//common:value_kind", - "//common:value_testing", - "//internal:testing", - "//internal:testing_descriptor_pool", - "//internal:testing_message_factory", - "//runtime:activation", - "//runtime:runtime_options", - "//runtime/internal:errors", - "//runtime/internal:runtime_type_provider", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/status", - "@com_google_protobuf//:protobuf", - ], -) - cc_library( name = "iterator_stack", hdrs = ["iterator_stack.h"], diff --git a/eval/eval/cel_expression_flat_impl.cc b/eval/eval/cel_expression_flat_impl.cc index c0f0b5e71..56b4d32a2 100644 --- a/eval/eval/cel_expression_flat_impl.cc +++ b/eval/eval/cel_expression_flat_impl.cc @@ -16,19 +16,12 @@ #include #include -#include #include "absl/base/nullability.h" #include "absl/log/absl_check.h" -#include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" -#include "common/native_type.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/internal/adapter_activation_impl.h" #include "eval/internal/interop.h" @@ -37,7 +30,6 @@ #include "eval/public/cel_value.h" #include "internal/casts.h" #include "internal/status_macros.h" -#include "runtime/internal/runtime_env.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -46,7 +38,6 @@ namespace google::api::expr::runtime { namespace { using ::cel::Value; -using ::cel::runtime_internal::RuntimeEnv; EvaluationListener AdaptListener(const CelEvaluationListener& listener) { if (!listener) return nullptr; @@ -123,58 +114,4 @@ std::unique_ptr CelExpressionFlatImpl::CreateState() const { flat_expression_); } -absl::StatusOr> -CelExpressionRecursiveImpl::Create( - absl_nonnull std::shared_ptr env, - FlatExpression flat_expr) { - const ExpressionStepLogic* logic = nullptr; - if (!flat_expr.path().empty() && flat_expr.path()[0].IsGenericStep()) { - logic = flat_expr.path()[0].GetGenericStep(); - } - if (logic != nullptr && - logic->GetNativeTypeId() != cel::NativeTypeId::For()) { - return absl::InvalidArgumentError(absl::StrCat( - "Expected a recursive program step", flat_expr.path().size())); - } - - auto* instance = - new CelExpressionRecursiveImpl(std::move(env), std::move(flat_expr)); - - return absl::WrapUnique(instance); -} - -absl::StatusOr CelExpressionRecursiveImpl::Trace( - const BaseActivation& activation, google::protobuf::Arena* arena, - CelEvaluationListener callback, CelEvaluationState* state) const { - std::unique_ptr inline_state; - if (state == nullptr) { - inline_state = CreateState(); - state = inline_state.get(); - } - auto derived_state = ::cel::internal::down_cast(state); - if (arena != nullptr) { - derived_state->Rebind(arena); - } else { - arena = derived_state->arena(); - } - if (state != inline_state.get()) { - derived_state->comprehension_slots().Reset(); - } - ABSL_DCHECK(arena != nullptr) - << "arena must be implicitly provided when using InitializeState() or " - "explicitly provided when using CreateState()"; - cel::interop_internal::AdapterActivationImpl modern_activation(activation); - ExecutionFrameBase execution_frame( - modern_activation, AdaptListener(callback), flat_expression_.options(), - flat_expression_.type_provider(), env_->descriptor_pool.get(), - env_->MutableMessageFactory(), arena, - /*embedder_context=*/nullptr, derived_state->comprehension_slots()); - - cel::Value result; - AttributeTrail trail; - CEL_RETURN_IF_ERROR(root_->Evaluate(execution_frame, result, trail)); - - return cel::interop_internal::ModernValueToLegacyValueOrDie(arena, result); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/cel_expression_flat_impl.h b/eval/eval/cel_expression_flat_impl.h index 055147477..5e9320332 100644 --- a/eval/eval/cel_expression_flat_impl.h +++ b/eval/eval/cel_expression_flat_impl.h @@ -15,19 +15,15 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CEL_EXPRESSION_FLAT_IMPL_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CEL_EXPRESSION_FLAT_IMPL_H_ -#include #include #include #include "absl/base/nullability.h" #include "absl/status/statusor.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/public/base_activation.h" #include "eval/public/cel_expression.h" #include "eval/public/cel_value.h" -#include "internal/casts.h" #include "runtime/internal/runtime_env.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -98,83 +94,6 @@ class CelExpressionFlatImpl : public CelExpression { FlatExpression flat_expression_; }; -// Implementation of the CelExpression that evaluates a recursive representation -// of the AST. -// -// This class adapts FlatExpression to implement the CelExpression interface. -// -// Assumes that the flat expression is wrapping a simple recursive program. -class CelExpressionRecursiveImpl : public CelExpression { - private: - class EvaluationState : public CelEvaluationState { - public: - explicit EvaluationState(size_t comprehension_slots) - : EvaluationState(nullptr, comprehension_slots) {} - - EvaluationState(google::protobuf::Arena* arena, size_t comprehension_slots) - : arena_(arena), comprehension_slots_(comprehension_slots) {} - - google::protobuf::Arena* arena() { return arena_; } - - void Rebind(google::protobuf::Arena* arena) { arena_ = arena; } - - ComprehensionSlots& comprehension_slots() { return comprehension_slots_; } - - private: - google::protobuf::Arena* arena_; - ComprehensionSlots comprehension_slots_; - }; - - public: - static absl::StatusOr> Create( - absl_nonnull std::shared_ptr env, - FlatExpression flat_expression); - - // Move-only - CelExpressionRecursiveImpl(const CelExpressionRecursiveImpl&) = delete; - CelExpressionRecursiveImpl& operator=(const CelExpressionRecursiveImpl&) = - delete; - CelExpressionRecursiveImpl(CelExpressionRecursiveImpl&&) = default; - CelExpressionRecursiveImpl& operator=(CelExpressionRecursiveImpl&&) = delete; - - // Implement CelExpression. - std::unique_ptr InitializeState( - google::protobuf::Arena* arena) const override { - return std::make_unique( - arena, flat_expression_.comprehension_slots_size()); - } - - // Implement CelExpression. - std::unique_ptr CreateState() const override { - return std::make_unique( - flat_expression_.comprehension_slots_size()); - } - - absl::StatusOr Trace(const BaseActivation& activation, - google::protobuf::Arena* arena, - CelEvaluationListener callback, - CelEvaluationState* state) const override; - - // Exposed for inspection in tests. - const FlatExpression& flat_expression() const { return flat_expression_; } - - const DirectExpressionStep* root() const { return root_; } - - private: - explicit CelExpressionRecursiveImpl( - absl_nonnull std::shared_ptr env, - FlatExpression flat_expression) - : env_(std::move(env)), - flat_expression_(std::move(flat_expression)), - root_(cel::internal::down_cast( - flat_expression_.path()[0].GetGenericStep()) - ->wrapped()) {} - - absl_nonnull std::shared_ptr env_; - FlatExpression flat_expression_; - const DirectExpressionStep* root_; -}; - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_CEL_EXPRESSION_FLAT_IMPL_H_ diff --git a/eval/eval/compiler_constant_step.cc b/eval/eval/compiler_constant_step.cc deleted file mode 100644 index c1b46b92e..000000000 --- a/eval/eval/compiler_constant_step.cc +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "eval/eval/compiler_constant_step.h" - -#include "absl/status/status.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/evaluator_core.h" - -namespace google::api::expr::runtime { - -using ::cel::Value; - -absl::Status DirectCompilerConstantStep::Evaluate( - ExecutionFrameBase& frame, Value& result, AttributeTrail& attribute) const { - result = value_; - return absl::OkStatus(); -} - -} // namespace google::api::expr::runtime diff --git a/eval/eval/compiler_constant_step.h b/eval/eval/compiler_constant_step.h deleted file mode 100644 index 6708e02de..000000000 --- a/eval/eval/compiler_constant_step.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPILER_CONSTANT_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPILER_CONSTANT_STEP_H_ - -#include -#include - -#include "absl/status/status.h" -#include "common/native_type.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" - -namespace google::api::expr::runtime { - -// DirectExpressionStep implementation that simply assigns a constant value. -// -// Overrides NativeTypeId() allow the FlatExprBuilder and extensions to -// inspect the underlying value. -class DirectCompilerConstantStep : public DirectExpressionStep { - public: - DirectCompilerConstantStep(cel::Value value, int64_t expr_id) - : DirectExpressionStep(expr_id), value_(std::move(value)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute) const override; - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - const cel::Value& value() const { return value_; } - - private: - cel::Value value_; -}; - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_COMPILER_CONSTANT_STEP_H_ diff --git a/eval/eval/compiler_constant_step_test.cc b/eval/eval/compiler_constant_step_test.cc deleted file mode 100644 index 181728479..000000000 --- a/eval/eval/compiler_constant_step_test.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "eval/eval/compiler_constant_step.h" - -#include "common/native_type.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" -#include "runtime/internal/runtime_type_provider.h" -#include "runtime/runtime_options.h" -#include "google/protobuf/arena.h" - -namespace google::api::expr::runtime { - -namespace { - -class DirectCompilerConstantStepTest : public testing::Test { - public: - DirectCompilerConstantStepTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()) {} - - protected: - google::protobuf::Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; - cel::Activation empty_activation_; - cel::RuntimeOptions options_; -}; - -TEST_F(DirectCompilerConstantStepTest, Evaluate) { - ExecutionFrameBase frame(empty_activation_, options_, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - DirectCompilerConstantStep step(cel::IntValue(42), -1); - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step.Evaluate(frame, result, attr), absl_testing::IsOk()); - - EXPECT_EQ(result.GetInt().NativeValue(), 42); -} - -TEST_F(DirectCompilerConstantStepTest, TypeId) { - DirectCompilerConstantStep step(cel::IntValue(42), -1); - - const DirectExpressionStep& abstract_step = step; - EXPECT_EQ(abstract_step.GetNativeTypeId(), - cel::NativeTypeId::For()); -} - -TEST_F(DirectCompilerConstantStepTest, Value) { - DirectCompilerConstantStep step(cel::IntValue(42), -1); - - EXPECT_EQ(step.value().GetInt().NativeValue(), 42); -} - -} // namespace -} // namespace google::api::expr::runtime diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc index 661a8f2e4..321e9c213 100644 --- a/eval/eval/comprehension_step.cc +++ b/eval/eval/comprehension_step.cc @@ -1,13 +1,11 @@ #include "eval/eval/comprehension_step.h" #include -#include #include #include #include "absl/base/attributes.h" #include "absl/base/casts.h" -#include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" @@ -17,11 +15,9 @@ #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/iterator_stack.h" #include "eval/internal/errors.h" -#include "internal/status_macros.h" namespace google::api::expr::runtime { namespace { @@ -34,7 +30,6 @@ enum class IterableKind { using ::cel::AttributeQualifier; using ::cel::UnknownValue; using ::cel::Value; -using ::cel::ValueIterator; using ::cel::ValueIteratorPtr; using ::cel::ValueKind; using ::cel::runtime_internal::CreateNoMatchingOverloadError; @@ -56,373 +51,6 @@ AttributeQualifier AttributeQualifierFromValue(const Value& v) { } } -class ComprehensionDirectStep final : public DirectExpressionStep { - public: - explicit ComprehensionDirectStep( - size_t iter_slot, size_t iter2_slot, size_t accu_slot, - std::unique_ptr range, - std::unique_ptr accu_init, - std::unique_ptr loop_step, - std::unique_ptr condition_step, - std::unique_ptr result_step, bool shortcircuiting, - int64_t expr_id) - : DirectExpressionStep(expr_id), - iter_slot_(iter_slot), - iter2_slot_(iter2_slot), - accu_slot_(accu_slot), - range_(std::move(range)), - accu_init_(std::move(accu_init)), - loop_step_(std::move(loop_step)), - condition_(std::move(condition_step)), - result_step_(std::move(result_step)), - shortcircuiting_(shortcircuiting) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const override { - return iter_slot_ == iter2_slot_ ? Evaluate1(frame, result, trail) - : Evaluate2(frame, result, trail); - } - - private: - absl::Status Evaluate1(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const; - - absl::StatusOr Evaluate1Unknown( - ExecutionFrameBase& frame, IterableKind range_iter_kind, - const AttributeTrail& range_iter_attr, - ValueIterator* absl_nonnull range_iter, - ComprehensionSlots::Slot* absl_nonnull accu_slot, - ComprehensionSlots::Slot* absl_nonnull iter_slot, Value& result, - AttributeTrail& trail) const; - - absl::StatusOr Evaluate1Known( - ExecutionFrameBase& frame, ValueIterator* absl_nonnull range_iter, - ComprehensionSlots::Slot* absl_nonnull accu_slot, - ComprehensionSlots::Slot* absl_nonnull iter_slot, Value& result, - AttributeTrail& trail) const; - - absl::Status Evaluate2(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const; - - const size_t iter_slot_; - const size_t iter2_slot_; - const size_t accu_slot_; - const std::unique_ptr range_; - const std::unique_ptr accu_init_; - const std::unique_ptr loop_step_; - const std::unique_ptr condition_; - const std::unique_ptr result_step_; - const bool shortcircuiting_; -}; - -absl::Status ComprehensionDirectStep::Evaluate1(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& trail) const { - Value range; - AttributeTrail range_attr; - CEL_RETURN_IF_ERROR(range_->Evaluate(frame, range, range_attr)); - - if (frame.unknown_processing_enabled() && range.IsMap()) { - if (frame.attribute_utility().CheckForUnknownPartial(range_attr)) { - result = - frame.attribute_utility().CreateUnknownSet(range_attr.attribute()); - return absl::OkStatus(); - } - } - - absl_nullability_unknown ValueIteratorPtr range_iter; - IterableKind iterable_kind; - switch (range.kind()) { - case ValueKind::kList: { - CEL_ASSIGN_OR_RETURN(range_iter, range.GetList().NewIterator()); - iterable_kind = IterableKind::kList; - } break; - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN(range_iter, range.GetMap().NewIterator()); - iterable_kind = IterableKind::kMap; - } break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(range); - return absl::OkStatus(); - default: - result = cel::ErrorValue::From( - CreateNoMatchingOverloadError(""), frame.arena()); - return absl::OkStatus(); - } - ABSL_DCHECK(range_iter != nullptr); - - ComprehensionSlots::Slot* accu_slot = - frame.comprehension_slots().Get(accu_slot_); - ABSL_DCHECK(accu_slot != nullptr); - - { - Value accu_init; - AttributeTrail accu_init_attr; - CEL_RETURN_IF_ERROR(accu_init_->Evaluate(frame, accu_init, accu_init_attr)); - accu_slot->Set(std::move(accu_init), std::move(accu_init_attr)); - } - - ComprehensionSlots::Slot* iter_slot = - frame.comprehension_slots().Get(iter_slot_); - ABSL_DCHECK(iter_slot != nullptr); - iter_slot->Set(); - - bool should_skip_result; - if (frame.unknown_processing_enabled()) { - CEL_ASSIGN_OR_RETURN( - should_skip_result, - Evaluate1Unknown(frame, iterable_kind, range_attr, range_iter.get(), - accu_slot, iter_slot, result, trail)); - } else { - CEL_ASSIGN_OR_RETURN(should_skip_result, - Evaluate1Known(frame, range_iter.get(), accu_slot, - iter_slot, result, trail)); - } - - frame.comprehension_slots().ClearSlot(iter_slot_); - if (!should_skip_result) { - CEL_RETURN_IF_ERROR(result_step_->Evaluate(frame, result, trail)); - } - frame.comprehension_slots().ClearSlot(accu_slot_); - return absl::OkStatus(); -} - -absl::StatusOr ComprehensionDirectStep::Evaluate1Unknown( - ExecutionFrameBase& frame, IterableKind range_iter_kind, - const AttributeTrail& range_iter_attr, - ValueIterator* absl_nonnull range_iter, - ComprehensionSlots::Slot* absl_nonnull accu_slot, - ComprehensionSlots::Slot* absl_nonnull iter_slot, Value& result, - AttributeTrail& trail) const { - Value condition; - AttributeTrail condition_attr; - Value key_or_value; - Value* key; - Value* value; - - switch (range_iter_kind) { - case IterableKind::kList: - key = &key_or_value; - value = iter_slot->mutable_value(); - break; - case IterableKind::kMap: - key = iter_slot->mutable_value(); - value = nullptr; - break; - default: - ABSL_UNREACHABLE(); - } - while (true) { - CEL_ASSIGN_OR_RETURN(bool ok, range_iter->Next2(frame.descriptor_pool(), - frame.message_factory(), - frame.arena(), key, value)); - if (!ok) { - break; - } - CEL_RETURN_IF_ERROR(frame.IncrementIterations()); - *iter_slot->mutable_attribute() = - range_iter_attr.Step(AttributeQualifierFromValue(*key)); - if (frame.attribute_utility().CheckForUnknownExact( - iter_slot->attribute())) { - *iter_slot->mutable_value() = frame.attribute_utility().CreateUnknownSet( - iter_slot->attribute().attribute()); - } - - // Evaluate the loop condition. - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - - switch (condition.kind()) { - case ValueKind::kBool: - break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(condition); - return true; - default: - result = cel::ErrorValue::From( - CreateNoMatchingOverloadError(""), frame.arena()); - return true; - } - - if (shortcircuiting_ && !absl::implicit_cast(condition.GetBool())) { - break; - } - - // Evaluate the loop step. - CEL_RETURN_IF_ERROR(loop_step_->Evaluate(frame, *accu_slot->mutable_value(), - *accu_slot->mutable_attribute())); - } - return false; -} - -absl::StatusOr ComprehensionDirectStep::Evaluate1Known( - ExecutionFrameBase& frame, ValueIterator* absl_nonnull range_iter, - ComprehensionSlots::Slot* absl_nonnull accu_slot, - ComprehensionSlots::Slot* absl_nonnull iter_slot, Value& result, - AttributeTrail& trail) const { - Value condition; - AttributeTrail condition_attr; - - while (true) { - CEL_ASSIGN_OR_RETURN( - bool ok, - range_iter->Next1(frame.descriptor_pool(), frame.message_factory(), - frame.arena(), iter_slot->mutable_value())); - if (!ok) { - break; - } - CEL_RETURN_IF_ERROR(frame.IncrementIterations()); - - // Evaluate the loop condition. - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - - switch (condition.kind()) { - case ValueKind::kBool: - break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(condition); - return true; - default: - result = cel::ErrorValue::From( - CreateNoMatchingOverloadError(""), frame.arena()); - return true; - } - - if (shortcircuiting_ && !absl::implicit_cast(condition.GetBool())) { - break; - } - - // Evaluate the loop step. - CEL_RETURN_IF_ERROR(loop_step_->Evaluate(frame, *accu_slot->mutable_value(), - *accu_slot->mutable_attribute())); - } - return false; -} - -absl::Status ComprehensionDirectStep::Evaluate2(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& trail) const { - Value range; - AttributeTrail range_attr; - CEL_RETURN_IF_ERROR(range_->Evaluate(frame, range, range_attr)); - - if (frame.unknown_processing_enabled() && range.IsMap()) { - if (frame.attribute_utility().CheckForUnknownPartial(range_attr)) { - result = - frame.attribute_utility().CreateUnknownSet(range_attr.attribute()); - return absl::OkStatus(); - } - } - - absl_nullability_unknown ValueIteratorPtr range_iter; - switch (range.kind()) { - case ValueKind::kList: { - CEL_ASSIGN_OR_RETURN(range_iter, range.GetList().NewIterator()); - } break; - case ValueKind::kMap: { - CEL_ASSIGN_OR_RETURN(range_iter, range.GetMap().NewIterator()); - } break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(range); - return absl::OkStatus(); - default: - result = cel::ErrorValue::From( - CreateNoMatchingOverloadError(""), frame.arena()); - return absl::OkStatus(); - } - ABSL_DCHECK(range_iter != nullptr); - - ComprehensionSlots::Slot* accu_slot = - frame.comprehension_slots().Get(accu_slot_); - ABSL_DCHECK(accu_slot != nullptr); - - { - Value accu_init; - AttributeTrail accu_init_attr; - CEL_RETURN_IF_ERROR(accu_init_->Evaluate(frame, accu_init, accu_init_attr)); - accu_slot->Set(std::move(accu_init), std::move(accu_init_attr)); - } - - ComprehensionSlots::Slot* iter_slot = - frame.comprehension_slots().Get(iter_slot_); - ABSL_DCHECK(iter_slot != nullptr); - iter_slot->Set(); - - ComprehensionSlots::Slot* iter2_slot = - frame.comprehension_slots().Get(iter2_slot_); - ABSL_DCHECK(iter2_slot != nullptr); - iter2_slot->Set(); - - Value condition; - AttributeTrail condition_attr; - bool should_skip_result = false; - - while (true) { - CEL_ASSIGN_OR_RETURN( - bool ok, - range_iter->Next2(frame.descriptor_pool(), frame.message_factory(), - frame.arena(), iter_slot->mutable_value(), - iter2_slot->mutable_value())); - if (!ok) { - break; - } - CEL_RETURN_IF_ERROR(frame.IncrementIterations()); - if (frame.unknown_processing_enabled()) { - *iter_slot->mutable_attribute() = *iter2_slot->mutable_attribute() = - range_attr.Step(AttributeQualifierFromValue(iter_slot->value())); - if (frame.attribute_utility().CheckForUnknownExact( - iter_slot->attribute())) { - *iter2_slot->mutable_value() = - frame.attribute_utility().CreateUnknownSet( - iter_slot->attribute().attribute()); - } - } - - // Evaluate the loop condition. - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - - switch (condition.kind()) { - case ValueKind::kBool: - break; - case ValueKind::kError: - ABSL_FALLTHROUGH_INTENDED; - case ValueKind::kUnknown: - result = std::move(condition); - should_skip_result = true; - goto finish; - default: - result = cel::ErrorValue::From( - CreateNoMatchingOverloadError(""), frame.arena()); - should_skip_result = true; - goto finish; - } - - if (shortcircuiting_ && !absl::implicit_cast(condition.GetBool())) { - break; - } - - // Evaluate the loop step. - CEL_RETURN_IF_ERROR(loop_step_->Evaluate(frame, *accu_slot->mutable_value(), - *accu_slot->mutable_attribute())); - } - -finish: - iter_slot->Clear(); - iter2_slot->Clear(); - if (!should_skip_result) { - CEL_RETURN_IF_ERROR(result_step_->Evaluate(frame, result, trail)); - } - accu_slot->Clear(); - return absl::OkStatus(); -} - } // namespace void ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { @@ -717,20 +345,6 @@ void ComprehensionCondStep::Evaluate2(ExecutionFrame* frame) const { } } -std::unique_ptr CreateDirectComprehensionStep( - size_t iter_slot, size_t iter2_slot, size_t accu_slot, - std::unique_ptr range, - std::unique_ptr accu_init, - std::unique_ptr loop_step, - std::unique_ptr condition_step, - std::unique_ptr result_step, bool shortcircuiting, - int64_t expr_id) { - return std::make_unique( - iter_slot, iter2_slot, accu_slot, std::move(range), std::move(accu_init), - std::move(loop_step), std::move(condition_step), std::move(result_step), - shortcircuiting, expr_id); -} - void EvaluateComprehensionFinishStep(size_t accu_slot, ExecutionFrame& frame) { if (!frame.value_stack().HasEnough(2)) { frame.Abort( diff --git a/eval/eval/comprehension_step.h b/eval/eval/comprehension_step.h index 8bac8dab3..809b94d13 100644 --- a/eval/eval/comprehension_step.h +++ b/eval/eval/comprehension_step.h @@ -4,12 +4,8 @@ #include #include #include -#include -#include "absl/status/status.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/expression_step_logic.h" -#include "eval/eval/iterator_stack.h" namespace google::api::expr::runtime { @@ -84,16 +80,6 @@ class ComprehensionCondStep final { int32_t error_jump_offset_ = std::numeric_limits::max(); }; -// Creates a step for executing a comprehension. -std::unique_ptr CreateDirectComprehensionStep( - size_t iter_slot, size_t iter2_slot, size_t accu_slot, - std::unique_ptr range, - std::unique_ptr accu_init, - std::unique_ptr loop_step, - std::unique_ptr condition_step, - std::unique_ptr result_step, bool shortcircuiting, - int64_t expr_id); - // Runs a cleanup step for the comprehension. // Removes the comprehension context then pushes the 'result' sub expression to // the top of the stack. diff --git a/eval/eval/comprehension_step_test.cc b/eval/eval/comprehension_step_test.cc index 09c4eb228..5964421ed 100644 --- a/eval/eval/comprehension_step_test.cc +++ b/eval/eval/comprehension_step_test.cc @@ -8,17 +8,10 @@ #include "cel/expr/syntax.pb.h" #include "google/protobuf/struct.pb.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "base/type_provider.h" #include "common/expr.h" -#include "common/value.h" -#include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" #include "eval/eval/ident_step.h" @@ -26,13 +19,8 @@ #include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" #include "eval/public/structs/cel_proto_wrapper.h" -#include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -40,19 +28,12 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::absl_testing::StatusIs; -using ::cel::BoolValue; using ::cel::Expr; -using ::cel::IntValue; using ::cel::TypeProvider; -using ::cel::Value; using ::cel::runtime_internal::NewTestingRuntimeEnv; -using ::cel::test::BoolValueIs; using ::google::protobuf::Struct; using ::google::protobuf::Arena; -using ::testing::_; using ::testing::Eq; -using ::testing::Return; using ::testing::SizeIs; class ListKeysStepTest : public testing::Test { @@ -184,305 +165,5 @@ TEST_F(ListKeysStepTest, UnknownSetPassedThrough) { EXPECT_THAT(eval_result->UnknownSetOrDie()->unknown_attributes(), SizeIs(1)); } -class MockDirectStep : public DirectExpressionStep { - public: - MockDirectStep() : DirectExpressionStep(-1) {} - - MOCK_METHOD(absl::Status, Evaluate, - (ExecutionFrameBase&, Value&, AttributeTrail&), - (const, override)); -}; - -// Test fixture for comprehensions. -// -// Comprehensions are quite involved so tests here focus on edge cases that are -// hard to exercise normally in functional-style tests for the planner. -class DirectComprehensionTest : public testing::Test { - public: - DirectComprehensionTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()), slots_(2) {} - - // returns a two element list for testing [1, 2]. - absl::StatusOr MakeList() { - auto builder = cel::NewListValueBuilder(&arena_); - - CEL_RETURN_IF_ERROR(builder->Add(IntValue(1))); - CEL_RETURN_IF_ERROR(builder->Add(IntValue(2))); - return std::move(*builder).Build(); - } - - protected: - google::protobuf::Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; - ComprehensionSlots slots_; - cel::Activation empty_activation_; -}; - -TEST_F(DirectComprehensionTest, PropagateRangeNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto range_step = std::make_unique(); - MockDirectStep* mock = range_step.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test range error"))); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/std::move(range_step), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test range error")); -} - -TEST_F(DirectComprehensionTest, PropagateAccuInitNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto accu_init = std::make_unique(); - MockDirectStep* mock = accu_init.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test accu init error"))); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/std::move(accu_init), - /*loop_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test accu init error")); -} - -TEST_F(DirectComprehensionTest, PropagateLoopNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto loop_step = std::make_unique(); - MockDirectStep* mock = loop_step.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test loop error"))); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/std::move(loop_step), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test loop error")); -} - -TEST_F(DirectComprehensionTest, PropagateConditionNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto condition = std::make_unique(); - MockDirectStep* mock = condition.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test condition error"))); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*condition_step=*/std::move(condition), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test condition error")); -} - -TEST_F(DirectComprehensionTest, PropagateResultNonOkStatus) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto result_step = std::make_unique(); - MockDirectStep* mock = result_step.get(); - - ON_CALL(*mock, Evaluate(_, _, _)) - .WillByDefault(Return(absl::InternalError("test result error"))); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/std::move(result_step), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal, "test result error")); -} - -TEST_F(DirectComprehensionTest, Shortcircuit) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto loop_step = std::make_unique(); - MockDirectStep* mock = loop_step.get(); - - EXPECT_CALL(*mock, Evaluate(_, _, _)) - .Times(0) - .WillRepeatedly([](ExecutionFrameBase&, Value& result, AttributeTrail&) { - result = BoolValue(false); - return absl::OkStatus(); - }); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/std::move(loop_step), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - ASSERT_THAT(compre_step->Evaluate(frame, result, trail), IsOk()); - EXPECT_THAT(result, BoolValueIs(false)); -} - -TEST_F(DirectComprehensionTest, IterationLimit) { - cel::RuntimeOptions options; - options.comprehension_max_iterations = 2; - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto loop_step = std::make_unique(); - MockDirectStep* mock = loop_step.get(); - - EXPECT_CALL(*mock, Evaluate(_, _, _)) - .Times(1) - .WillRepeatedly([](ExecutionFrameBase&, Value& result, AttributeTrail&) { - result = BoolValue(false); - return absl::OkStatus(); - }); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/std::move(loop_step), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(true)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/true, -1); - - Value result; - AttributeTrail trail; - EXPECT_THAT(compre_step->Evaluate(frame, result, trail), - StatusIs(absl::StatusCode::kInternal)); -} - -TEST_F(DirectComprehensionTest, Exhaustive) { - cel::RuntimeOptions options; - - ExecutionFrameBase frame(empty_activation_, /*callback=*/nullptr, options, - type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_, - /*embedder_context=*/nullptr, slots_); - - auto loop_step = std::make_unique(); - MockDirectStep* mock = loop_step.get(); - - EXPECT_CALL(*mock, Evaluate(_, _, _)) - .Times(2) - .WillRepeatedly([](ExecutionFrameBase&, Value& result, AttributeTrail&) { - result = BoolValue(false); - return absl::OkStatus(); - }); - - ASSERT_OK_AND_ASSIGN(auto list, MakeList()); - - auto compre_step = CreateDirectComprehensionStep( - 0, 0, 1, - /*range_step=*/CreateConstValueDirectStep(std::move(list)), - /*accu_init=*/CreateConstValueDirectStep(BoolValue(false)), - /*loop_step=*/std::move(loop_step), - /*condition_step=*/CreateConstValueDirectStep(BoolValue(false)), - /*result_step=*/CreateDirectSlotIdentStep("__result__", 1, -1), - /*shortcircuiting=*/false, -1); - - Value result; - AttributeTrail trail; - ASSERT_THAT(compre_step->Evaluate(frame, result, trail), IsOk()); - EXPECT_THAT(result, BoolValueIs(false)); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/const_value_step.h b/eval/eval/const_value_step.h deleted file mode 100644 index 55425ceab..000000000 --- a/eval/eval/const_value_step.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONST_VALUE_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONST_VALUE_STEP_H_ - -#include -#include -#include - -#include "absl/status/statusor.h" -#include "common/value.h" -#include "eval/eval/compiler_constant_step.h" -#include "eval/eval/direct_expression_step.h" - -namespace google::api::expr::runtime { - -// Factory method for Constant AST node expression recursive step. -inline std::unique_ptr CreateConstValueDirectStep( - cel::Value value, int64_t id = -1) { - return std::make_unique(std::move(value), id); -} - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONST_VALUE_STEP_H_ diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index dc1efbd12..5f7cf3e2b 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -17,12 +17,11 @@ #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" #include "eval/eval/attribute_utility.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" #include "internal/number.h" -#include "internal/status_macros.h" #include "runtime/internal/errors.h" namespace google::api::expr::runtime { @@ -311,55 +310,8 @@ void ContainerAccessStep::Evaluate(ExecutionFrame* frame) const { std::move(result), std::move(result_trail)); } -class DirectContainerAccessStep : public DirectExpressionStep { - public: - DirectContainerAccessStep( - std::unique_ptr container_step, - std::unique_ptr key_step, - bool enable_optional_types, int64_t expr_id) - : DirectExpressionStep(expr_id), - container_step_(std::move(container_step)), - key_step_(std::move(key_step)), - enable_optional_types_(enable_optional_types) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const override; - - private: - std::unique_ptr container_step_; - std::unique_ptr key_step_; - bool enable_optional_types_; -}; - -absl::Status DirectContainerAccessStep::Evaluate(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& trail) const { - Value container; - Value key; - AttributeTrail container_trail; - AttributeTrail key_trail; - - CEL_RETURN_IF_ERROR( - container_step_->Evaluate(frame, container, container_trail)); - CEL_RETURN_IF_ERROR(key_step_->Evaluate(frame, key, key_trail)); - - PerformLookup(frame, container, key, container_trail, enable_optional_types_, - result, trail); - - return absl::OkStatus(); -} - } // namespace -std::unique_ptr CreateDirectContainerAccessStep( - std::unique_ptr container_step, - std::unique_ptr key_step, bool enable_optional_types, - int64_t expr_id) { - return std::make_unique( - std::move(container_step), std::move(key_step), enable_optional_types, - expr_id); -} - // Factory method for Select - based Execution step absl::StatusOr> CreateContainerAccessStep( const cel::CallExpr& call, bool enable_optional_types) { diff --git a/eval/eval/container_access_step.h b/eval/eval/container_access_step.h index 5d5fdde81..38d4b34d4 100644 --- a/eval/eval/container_access_step.h +++ b/eval/eval/container_access_step.h @@ -1,21 +1,14 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONTAINER_ACCESS_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CONTAINER_ACCESS_STEP_H_ -#include #include #include "absl/status/statusor.h" #include "common/expr.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -std::unique_ptr CreateDirectContainerAccessStep( - std::unique_ptr container_step, - std::unique_ptr key_step, bool enable_optional_types, - int64_t expr_id); - // Factory method for Select - based Execution step absl::StatusOr> CreateContainerAccessStep( const cel::CallExpr& call, bool enable_optional_types = false); diff --git a/eval/eval/container_access_step_test.cc b/eval/eval/container_access_step_test.cc index 612732b43..a298c28e3 100644 --- a/eval/eval/container_access_step_test.cc +++ b/eval/eval/container_access_step_test.cc @@ -16,9 +16,7 @@ #include "common/ast.h" #include "common/expr.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_expr_builder_factory.h" @@ -34,6 +32,7 @@ #include "parser/parser.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" +#include "runtime/runtime_options.h" #include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -76,19 +75,10 @@ CelValue EvaluateAttributeHelper( container_expr.mutable_ident_expr().set_name("container"); key_expr.mutable_ident_expr().set_name("key"); - if (use_recursive_impl) { - path.push_back(ExpressionStep::MakeGenericStep( - std::make_unique(CreateDirectContainerAccessStep( - CreateDirectIdentStep("container", 1), - CreateDirectIdentStep("key", 2), - /*enable_optional_types=*/false, 3)), - 3)); - } else { path.push_back(ExpressionStep::MakeIdentifierStep("container", 1)); path.push_back(ExpressionStep::MakeIdentifierStep("key", 2)); path.push_back(ExpressionStep::MakeGenericStep( std::move(CreateContainerAccessStep(call).value()), 3)); - } cel::RuntimeOptions options; options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index 8a7470fa7..1d57fafa7 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -13,11 +13,10 @@ #include "common/expr.h" #include "common/value.h" #include "common/values/list_value_builder.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/attribute_utility.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "internal/status_macros.h" namespace google::api::expr::runtime { @@ -129,124 +128,8 @@ absl::flat_hash_set MakeOptionalIndicesSet( return optional_indices; } -class CreateListDirectStep : public DirectExpressionStep { - public: - CreateListDirectStep( - std::vector> elements, - absl::flat_hash_set optional_indices, int64_t expr_id) - : DirectExpressionStep(expr_id), - elements_(std::move(elements)), - optional_indices_(std::move(optional_indices)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override { - ListValueBuilderPtr builder = NewListValueBuilder(frame.arena()); - builder->Reserve(elements_.size()); - - AttributeUtility::Accumulator unknowns = - frame.attribute_utility().CreateAccumulator(); - AttributeTrail tmp_attr; - - for (size_t i = 0; i < elements_.size(); ++i) { - const auto& element = elements_[i]; - CEL_RETURN_IF_ERROR(element->Evaluate(frame, result, tmp_attr)); - - if (result.IsError()) { - return absl::OkStatus(); - } - - if (frame.attribute_tracking_enabled()) { - if (frame.missing_attribute_errors_enabled()) { - if (frame.attribute_utility().CheckForMissingAttribute(tmp_attr)) { - CEL_ASSIGN_OR_RETURN( - result, frame.attribute_utility().CreateMissingAttributeError( - tmp_attr.attribute(), frame.arena())); - return absl::OkStatus(); - } - } - if (frame.unknown_processing_enabled()) { - if (result.IsUnknown()) { - unknowns.Add(result.GetUnknown()); - } - if (frame.attribute_utility().CheckForUnknown(tmp_attr, - /*use_partial=*/true)) { - unknowns.Add(tmp_attr); - } - } - } - - if (!unknowns.IsEmpty()) { - // We found an unknown, there is no point in attempting to create a - // list. Instead iterate through the remaining elements and look for - // more unknowns. - continue; - } - - // Conditionally add if optional. - if (optional_indices_.contains(static_cast(i))) { - if (auto optional_arg = result.AsOptional(); optional_arg) { - if (!optional_arg->HasValue()) { - continue; - } - Value optional_arg_value; - optional_arg->Value(&optional_arg_value); - if (optional_arg_value.IsError()) { - // Error should never be in optional, but better safe than sorry. - result = std::move(optional_arg_value); - return absl::OkStatus(); - } - CEL_RETURN_IF_ERROR(builder->Add(std::move(optional_arg_value))); - continue; - } - result = cel::TypeConversionError(result.GetTypeName(), "optional_type", - frame.arena()); - return absl::OkStatus(); - } - - // Otherwise just add. - CEL_RETURN_IF_ERROR(builder->Add(std::move(result))); - } - - if (!unknowns.IsEmpty()) { - result = std::move(unknowns).Build(); - return absl::OkStatus(); - } - result = std::move(*builder).Build(); - - return absl::OkStatus(); - } - - private: - std::vector> elements_; - absl::flat_hash_set optional_indices_; -}; - -class DirectMutableListStep : public DirectExpressionStep { - public: - explicit DirectMutableListStep(int64_t expr_id) - : DirectExpressionStep(expr_id) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; -}; - -absl::Status DirectMutableListStep::Evaluate(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& attribute) const { - result = cel::CustomListValue( - cel::common_internal::NewMutableListValue(frame.arena()), frame.arena()); - return absl::OkStatus(); -} - } // namespace -std::unique_ptr CreateDirectListStep( - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id) { - return std::make_unique( - std::move(deps), std::move(optional_indices), expr_id); -} - absl::StatusOr> CreateCreateListStep( const cel::ListExpr& create_list_expr) { return std::make_unique( @@ -254,9 +137,4 @@ absl::StatusOr> CreateCreateListStep( MakeOptionalIndicesSet(create_list_expr)); } -std::unique_ptr CreateDirectMutableListStep( - int64_t expr_id) { - return std::make_unique(expr_id); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/create_list_step.h b/eval/eval/create_list_step.h index 3423febe5..9db49db32 100644 --- a/eval/eval/create_list_step.h +++ b/eval/eval/create_list_step.h @@ -1,34 +1,18 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CREATE_LIST_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CREATE_LIST_STEP_H_ -#include #include -#include -#include "absl/container/flat_hash_set.h" #include "absl/status/statusor.h" #include "common/expr.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -// Factory method for CreateList that evaluates recursively. -std::unique_ptr CreateDirectListStep( - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id); - // Factory method for CreateList which constructs an immutable list. absl::StatusOr> CreateCreateListStep( const cel::ListExpr& create_list_expr); -// Factory method for CreateList which constructs a mutable list. -// -// This is intended for the list construction step is generated for a -// list-building comprehension (rather than a user authored expression). -std::unique_ptr CreateDirectMutableListStep( - int64_t expr_id); - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_CREATE_LIST_STEP_H_ diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index 96da03ed2..a8f02b5d8 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -1,5 +1,6 @@ #include "eval/eval/create_list_step.h" +#include #include #include #include @@ -16,13 +17,8 @@ #include "base/type_provider.h" #include "common/expr.h" #include "common/value.h" -#include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/internal/interop.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" @@ -31,12 +27,8 @@ #include "eval/public/unknown_attribute_set.h" #include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -45,24 +37,14 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::absl_testing::IsOkAndHolds; -using ::absl_testing::StatusIs; using ::cel::Attribute; -using ::cel::AttributeQualifier; using ::cel::AttributeSet; -using ::cel::ErrorValue; using ::cel::Expr; -using ::cel::IntValue; using ::cel::TypeProvider; -using ::cel::UnknownValue; -using ::cel::Value; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; -using ::cel::test::IntValueIs; using ::testing::Eq; -using ::testing::HasSubstr; using ::testing::Not; -using ::testing::UnorderedElementsAre; // Helper method. Creates simple pipeline containing Select step and runs it. absl::StatusOr RunExpression( @@ -263,288 +245,6 @@ TEST(CreateListStepTest, CreateListHundredAnd2Unknowns) { EXPECT_THAT(result_set->unknown_attributes().size(), Eq(2)); } -TEST(CreateDirectListStep, Basic) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(IntValue(1), -1)); - deps.push_back(CreateConstValueDirectStep(IntValue(2), -1)); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsList()); - EXPECT_THAT(result.GetList().Size(), IsOkAndHolds(2)); -} - -TEST(CreateDirectListStep, ForwardFirstError) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue::From(absl::InternalError("test1"), &arena), -1)); - deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue::From(absl::InternalError("test2"), &arena), -1)); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInternal, "test1")); -} - -std::vector UnknownAttrNames(const UnknownValue& v) { - std::vector names; - names.reserve(v.ToAttributeSet().size()); - - for (const auto& attr : v.ToAttributeSet()) { - EXPECT_THAT(attr.AsString().status(), IsOk()); - names.push_back(attr.AsString().value_or("")); - } - - return names; -} - -TEST(CreateDirectListStep, MergeUnknowns) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - AttributeSet attr_set1({Attribute("var1")}); - AttributeSet attr_set2({Attribute("var2")}); - - std::vector> deps; - deps.push_back( - CreateConstValueDirectStep(cel::common_internal::MakeUnknownValue( - cel::Unknown(std::move(attr_set1))), - -1)); - deps.push_back( - CreateConstValueDirectStep(cel::common_internal::MakeUnknownValue( - cel::Unknown(std::move(attr_set2))), - -1)); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsUnknown()); - EXPECT_THAT(UnknownAttrNames(result.GetUnknown()), - UnorderedElementsAre("var1", "var2")); -} - -TEST(CreateDirectListStep, ErrorBeforeUnknown) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - AttributeSet attr_set1({Attribute("var1")}); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue::From(absl::InternalError("test1"), &arena), -1)); - deps.push_back(CreateConstValueDirectStep( - cel::ErrorValue::From(absl::InternalError("test2"), &arena), -1)); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInternal, "test1")); -} - -class SetAttrDirectStep : public DirectExpressionStep { - public: - explicit SetAttrDirectStep(Attribute attr) - : DirectExpressionStep(-1), attr_(std::move(attr)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attr) const override { - result = cel::NullValue(); - attr = AttributeTrail(attr_); - return absl::OkStatus(); - } - - private: - cel::Attribute attr_; -}; - -TEST(CreateDirectListStep, MissingAttribute) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - options.enable_missing_attribute_errors = true; - - ASSERT_THAT( - activation.SetMissingPatterns({cel::AttributePattern( - "var1", {cel::AttributeQualifierPattern::OfString("field1")})}), - IsOk()); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(cel::NullValue(), -1)); - deps.push_back(std::make_unique( - Attribute("var1", {AttributeQualifier::OfString("field1")}))); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT( - result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("var1.field1"))); -} - -TEST(CreateDirectListStep, OptionalPresentSet) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(IntValue(1), -1)); - deps.push_back(CreateConstValueDirectStep( - cel::OptionalValue::Of(IntValue(2), &arena), -1)); - auto step = CreateDirectListStep(std::move(deps), {1}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsList()); - auto list = result.GetList(); - EXPECT_THAT(list.Size(), IsOkAndHolds(2)); - EXPECT_THAT(list.Get(0, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena), - IsOkAndHolds(IntValueIs(1))); - EXPECT_THAT(list.Get(1, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena), - IsOkAndHolds(IntValueIs(2))); -} - -TEST(CreateDirectListStep, OptionalAbsentNotSet) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(IntValue(1), -1)); - deps.push_back(CreateConstValueDirectStep(cel::OptionalValue::None(), -1)); - auto step = CreateDirectListStep(std::move(deps), {1}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsList()); - auto list = result.GetList(); - EXPECT_THAT(list.Size(), IsOkAndHolds(1)); - EXPECT_THAT(list.Get(0, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena), - IsOkAndHolds(IntValueIs(1))); -} - -TEST(CreateDirectListStep, PartialUnknown) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - cel::Activation activation; - cel::RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - ASSERT_THAT( - activation.SetUnknownPatterns({cel::AttributePattern( - "var1", {cel::AttributeQualifierPattern::OfString("field1")})}), - IsOk()); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(cel::IntValue(1), -1)); - deps.push_back(std::make_unique(Attribute("var1", {}))); - auto step = CreateDirectListStep(std::move(deps), {}, -1); - - cel::Value result; - AttributeTrail attr; - - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsUnknown()); - EXPECT_THAT(UnknownAttrNames(result.GetUnknown()), - UnorderedElementsAre("var1")); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/create_map_step.cc b/eval/eval/create_map_step.cc index 15e696782..8283f1ed8 100644 --- a/eval/eval/create_map_step.cc +++ b/eval/eval/create_map_step.cc @@ -18,7 +18,6 @@ #include #include #include -#include #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" @@ -27,18 +26,15 @@ #include "absl/types/optional.h" #include "common/value.h" #include "common/values/map_value_builder.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "internal/status_macros.h" namespace google::api::expr::runtime { namespace { -using ::cel::ErrorValue; -using ::cel::ErrorValueAssign; using ::cel::ErrorValueReturn; using ::cel::MapValueBuilderPtr; using ::cel::UnknownValue; @@ -132,112 +128,6 @@ void CreateStructStepForMap::Evaluate(ExecutionFrame* frame) const { frame->value_stack().PopAndPush(2 * entry_count_, *std::move(result)); } -class DirectCreateMapStep : public DirectExpressionStep { - public: - DirectCreateMapStep(std::vector> deps, - absl::flat_hash_set optional_indices, - int64_t expr_id) - : DirectExpressionStep(expr_id), - deps_(std::move(deps)), - optional_indices_(std::move(optional_indices)), - entry_count_(deps_.size() / 2) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::vector> deps_; - absl::flat_hash_set optional_indices_; - size_t entry_count_; -}; - -absl::Status DirectCreateMapStep::Evaluate( - ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - auto unknowns = frame.attribute_utility().CreateAccumulator(); - - MapValueBuilderPtr builder = NewMapValueBuilder(frame.arena()); - builder->Reserve(entry_count_); - - for (size_t i = 0; i < entry_count_; i += 1) { - Value key; - Value value; - AttributeTrail tmp_attr; - int map_key_index = 2 * i; - int map_value_index = map_key_index + 1; - CEL_RETURN_IF_ERROR(deps_[map_key_index]->Evaluate(frame, key, tmp_attr)); - - if (key.IsError()) { - result = std::move(key); - return absl::OkStatus(); - } - - if (frame.unknown_processing_enabled()) { - if (key.IsUnknown()) { - unknowns.Add(key.GetUnknown()); - } else if (frame.attribute_utility().CheckForUnknownPartial(tmp_attr)) { - unknowns.Add(tmp_attr); - } - } - - CEL_RETURN_IF_ERROR(cel::CheckMapKey(key)) - .With(ErrorValueAssign(result, frame.arena())); - - CEL_RETURN_IF_ERROR( - deps_[map_value_index]->Evaluate(frame, value, tmp_attr)); - - if (value.IsError()) { - result = std::move(value); - return absl::OkStatus(); - } - - if (frame.unknown_processing_enabled()) { - if (value.IsUnknown()) { - unknowns.Add(value.GetUnknown()); - } else if (frame.attribute_utility().CheckForUnknownPartial(tmp_attr)) { - unknowns.Add(tmp_attr); - } - } - - // Preserve the stack machine behavior of forwarding unknowns before - // errors. - if (!unknowns.IsEmpty()) { - continue; - } - - if (optional_indices_.contains(static_cast(i))) { - if (auto optional_map_value = value.AsOptional(); optional_map_value) { - if (!optional_map_value->HasValue()) { - continue; - } - Value optional_map_value_value; - optional_map_value->Value(&optional_map_value_value); - if (optional_map_value_value.IsError()) { - // Error should never be in optional, but better safe than sorry. - result = optional_map_value_value; - return absl::OkStatus(); - } - CEL_RETURN_IF_ERROR( - builder->Put(std::move(key), std::move(optional_map_value_value))); - continue; - } - result = cel::TypeConversionError(value.DebugString(), "optional_type", - frame.arena()); - return absl::OkStatus(); - } - - CEL_RETURN_IF_ERROR(builder->Put(std::move(key), std::move(value))); - } - - if (!unknowns.IsEmpty()) { - result = std::move(unknowns).Build(); - return absl::OkStatus(); - } - - result = std::move(*builder).Build(); - return absl::OkStatus(); -} - class MutableMapStep final : public ExpressionStepBase { public: MutableMapStep() = default; @@ -248,28 +138,8 @@ class MutableMapStep final : public ExpressionStepBase { } }; -class DirectMutableMapStep final : public DirectExpressionStep { - public: - explicit DirectMutableMapStep(int64_t expr_id) - : DirectExpressionStep(expr_id) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - result = - cel::CustomMapValue(NewMutableMapValue(frame.arena()), frame.arena()); - return absl::OkStatus(); - } -}; - } // namespace -std::unique_ptr CreateDirectCreateMapStep( - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id) { - return std::make_unique( - std::move(deps), std::move(optional_indices), expr_id); -} - absl::StatusOr> CreateCreateStructStepForMap(size_t entry_count, absl::flat_hash_set optional_indices) { @@ -282,9 +152,4 @@ std::unique_ptr CreateMutableMapStep() { return std::make_unique(); } -std::unique_ptr CreateDirectMutableMapStep( - int64_t expr_id) { - return std::make_unique(expr_id); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/create_map_step.h b/eval/eval/create_map_step.h index 599a48534..2f9729999 100644 --- a/eval/eval/create_map_step.h +++ b/eval/eval/create_map_step.h @@ -18,23 +18,13 @@ #include #include #include -#include #include "absl/container/flat_hash_set.h" #include "absl/status/statusor.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -// Creates an expression step that evaluates a create map expression. -// -// Deps must have an even number of elements, that alternate key, value pairs. -// (key1, value1, key2, value2...). -std::unique_ptr CreateDirectCreateMapStep( - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id); - // Creates an `ExpressionStep` which performs `CreateStruct` for a map. absl::StatusOr> CreateCreateStructStepForMap(size_t entry_count, @@ -46,13 +36,6 @@ CreateCreateStructStepForMap(size_t entry_count, // map-building comprehension (rather than a user authored expression). std::unique_ptr CreateMutableMapStep(); -// Factory method for CreateMap which constructs a mutable map. -// -// This is intended for the map construction step is generated for a -// map-building comprehension (rather than a user authored expression). -std::unique_ptr CreateDirectMutableMapStep( - int64_t expr_id); - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_CREATE_MAP_STEP_H_ diff --git a/eval/eval/create_map_step_test.cc b/eval/eval/create_map_step_test.cc index d231db211..163fed860 100644 --- a/eval/eval/create_map_step_test.cc +++ b/eval/eval/create_map_step_test.cc @@ -29,9 +29,7 @@ #include "base/type_provider.h" #include "common/expr.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_value.h" #include "eval/public/unknown_set.h" @@ -87,33 +85,6 @@ absl::StatusOr CreateStackMachineProgram( return path; } -absl::StatusOr CreateRecursiveProgram( - const std::vector>& values, - Activation& activation) { - ExecutionPath path; - - int index = 0; - std::vector> deps; - for (const auto& item : values) { - std::string key_name = absl::StrCat("key", index); - std::string value_name = absl::StrCat("value", index); - - deps.push_back(CreateDirectIdentStep(key_name, -1)); - - deps.push_back(CreateDirectIdentStep(value_name, -1)); - - activation.InsertValue(key_name, item.first); - activation.InsertValue(value_name, item.second); - - index++; - } - path.push_back( - ExpressionStep::MakeGenericStep(std::make_unique( - CreateDirectCreateMapStep(std::move(deps), {}, -1)))); - - return path; -} - // Helper method. Creates simple pipeline containing CreateStruct step that // builds Map and runs it. // Equivalent to {key0: value0, ...} @@ -124,11 +95,7 @@ absl::StatusOr RunCreateMapExpression( Activation activation; ExecutionPath path; - if (enable_recursive_program) { - CEL_ASSIGN_OR_RETURN(path, CreateRecursiveProgram(values, activation)); - } else { - CEL_ASSIGN_OR_RETURN(path, CreateStackMachineProgram(values, activation)); - } + CEL_ASSIGN_OR_RETURN(path, CreateStackMachineProgram(values, activation)); cel::RuntimeOptions options; if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index 8fcdd3c5a..ef05a08f7 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -27,10 +27,9 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "internal/status_macros.h" namespace google::api::expr::runtime { @@ -143,125 +142,8 @@ void CreateStructStepForStruct::Evaluate(ExecutionFrame* frame) const { frame->value_stack().PopAndPush(entries_.size(), *std::move(result)); } -class DirectCreateStructStep : public DirectExpressionStep { - public: - DirectCreateStructStep( - int64_t expr_id, std::string name, std::vector field_keys, - std::vector> deps, - absl::flat_hash_set optional_indices) - : DirectExpressionStep(expr_id), - name_(std::move(name)), - field_keys_(std::move(field_keys)), - deps_(std::move(deps)), - optional_indices_(std::move(optional_indices)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& trail) const override; - - private: - std::string name_; - std::vector field_keys_; - std::vector> deps_; - absl::flat_hash_set optional_indices_; -}; - -absl::Status DirectCreateStructStep::Evaluate(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& trail) const { - Value field_value; - AttributeTrail field_attr; - auto unknowns = frame.attribute_utility().CreateAccumulator(); - - CEL_ASSIGN_OR_RETURN(auto builder, - frame.type_provider().NewValueBuilder( - name_, frame.message_factory(), frame.arena())); - if (builder == nullptr) { - result = cel::ErrorValue::From( - absl::NotFoundError(absl::StrCat("Unable to find builder: ", name_)), - frame.arena()); - return absl::OkStatus(); - } - - for (int i = 0; i < field_keys_.size(); i++) { - CEL_RETURN_IF_ERROR(deps_[i]->Evaluate(frame, field_value, field_attr)); - - // TODO(uncreated-issue/67): if the value is an error, we should be able to return - // early, however some client tests depend on the error message the struct - // impl returns in the stack machine version. - if (field_value.IsError()) { - result = std::move(field_value); - return absl::OkStatus(); - } - - if (frame.unknown_processing_enabled()) { - if (field_value.IsUnknown()) { - unknowns.Add(field_value.GetUnknown()); - } else if (frame.attribute_utility().CheckForUnknownPartial(field_attr)) { - unknowns.Add(field_attr); - } - } - - if (!unknowns.IsEmpty()) { - continue; - } - - if (optional_indices_.contains(static_cast(i))) { - if (auto optional_arg = field_value.AsOptional(); optional_arg) { - if (!optional_arg->HasValue()) { - continue; - } - Value optional_arg_value; - optional_arg->Value(&optional_arg_value); - if (optional_arg_value.IsError()) { - // Error should never be in optional, but better safe than sorry. - result = std::move(optional_arg_value); - return absl::OkStatus(); - } - CEL_ASSIGN_OR_RETURN( - absl::optional error_value, - builder->SetFieldByName(field_keys_[i], - std::move(optional_arg_value))); - if (error_value) { - result = std::move(*error_value); - return absl::OkStatus(); - } - continue; - } else { - result = cel::TypeConversionError(field_value.DebugString(), - "optional_type", frame.arena()); - return absl::OkStatus(); - } - } - - CEL_ASSIGN_OR_RETURN( - absl::optional error_value, - builder->SetFieldByName(field_keys_[i], std::move(field_value))); - if (error_value) { - result = std::move(*error_value); - return absl::OkStatus(); - } - } - - if (!unknowns.IsEmpty()) { - result = std::move(unknowns).Build(); - return absl::OkStatus(); - } - - CEL_ASSIGN_OR_RETURN(result, std::move(*builder).Build()); - return absl::OkStatus(); -} - } // namespace -std::unique_ptr CreateDirectCreateStructStep( - std::string resolved_name, std::vector field_keys, - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id) { - return std::make_unique( - expr_id, std::move(resolved_name), std::move(field_keys), std::move(deps), - std::move(optional_indices)); -} - std::unique_ptr CreateCreateStructStep( std::string name, std::vector field_keys, absl::flat_hash_set optional_indices) { diff --git a/eval/eval/create_struct_step.h b/eval/eval/create_struct_step.h index 11880493a..1dd42273a 100644 --- a/eval/eval/create_struct_step.h +++ b/eval/eval/create_struct_step.h @@ -21,18 +21,10 @@ #include #include "absl/container/flat_hash_set.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -// Creates an `ExpressionStep` which performs `CreateStruct` for a -// message/struct. -std::unique_ptr CreateDirectCreateStructStep( - std::string name, std::vector field_keys, - std::vector> deps, - absl::flat_hash_set optional_indices, int64_t expr_id); - // Creates an `ExpressionStep` which performs `CreateStruct` for a // message/struct. std::unique_ptr CreateCreateStructStep( diff --git a/eval/eval/create_struct_step_test.cc b/eval/eval/create_struct_step_test.cc index 06085c311..e348ecd20 100644 --- a/eval/eval/create_struct_step_test.cc +++ b/eval/eval/create_struct_step_test.cc @@ -29,11 +29,8 @@ #include "absl/strings/string_view.h" #include "absl/types/span.h" #include "base/type_provider.h" -#include "common/expr.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_type_registry.h" #include "eval/public/cel_value.h" @@ -82,25 +79,6 @@ absl::StatusOr MakeStackMachinePath(absl::string_view field) { return path; } -absl::StatusOr MakeRecursivePath(absl::string_view field) { - ExecutionPath path; - - std::vector> deps; - deps.push_back(CreateDirectIdentStep("message", -1)); - - auto step1 = - CreateDirectCreateStructStep("google.api.expr.runtime.TestMessage", - {std::string(field)}, std::move(deps), - /*optional_indices=*/{}, - - /*id=*/-1); - - path.push_back(ExpressionStep::MakeGenericStep( - std::make_unique(std::move(step1)))); - - return path; -} - // Helper method. Creates simple pipeline containing CreateStruct step that // builds message and runs it. absl::StatusOr RunExpression( @@ -122,11 +100,7 @@ absl::StatusOr RunExpression( } ExecutionPath path; - if (enable_recursive_planning) { - CEL_ASSIGN_OR_RETURN(path, MakeRecursivePath(field)); - } else { - CEL_ASSIGN_OR_RETURN(path, MakeStackMachinePath(field)); - } + CEL_ASSIGN_OR_RETURN(path, MakeStackMachinePath(field)); CelExpressionFlatImpl cel_expr( env, @@ -202,21 +176,11 @@ TEST_P(CreateCreateStructStepTest, TestEmptyMessageCreation) { env_->type_registry.GetComposedTypeProvider().FindType( "google.api.expr.runtime.TestMessage")); ASSERT_TRUE(maybe_type.has_value()); - if (enable_recursive_planning()) { - auto step = - CreateDirectCreateStructStep("google.api.expr.runtime.TestMessage", - /*fields=*/{}, - /*deps=*/{}, - /*optional_indices=*/{}, - /*id=*/-1); - path.push_back(ExpressionStep::MakeGenericStep( - std::make_unique(std::move(step)))); - } else { + auto step = CreateCreateStructStep("google.api.expr.runtime.TestMessage", /*fields=*/{}, /*optional_indices=*/{}); path.push_back(ExpressionStep::MakeGenericStep(std::move(step))); - } cel::RuntimeOptions options; if (enable_unknowns(), enable_recursive_planning()) { diff --git a/eval/eval/direct_expression_step.cc b/eval/eval/direct_expression_step.cc deleted file mode 100644 index 36b5ed1f5..000000000 --- a/eval/eval/direct_expression_step.cc +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "eval/eval/direct_expression_step.h" - -namespace google::api::expr::runtime { - -// Intentionally empty. - -} // namespace google::api::expr::runtime diff --git a/eval/eval/direct_expression_step.h b/eval/eval/direct_expression_step.h deleted file mode 100644 index 547adad41..000000000 --- a/eval/eval/direct_expression_step.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_DIRECT_EXPRESSION_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_DIRECT_EXPRESSION_STEP_H_ - -#include -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/types/optional.h" -#include "common/native_type.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" - -namespace google::api::expr::runtime { - -class ExecutionFrameBase; - -// Represents a directly evaluated CEL expression. -// -// Subexpressions assign to values on the C++ program stack and call their -// dependencies directly. -// -// This reduces the setup overhead for evaluation and minimizes value churn -// to / from a heap based value stack managed by the CEL runtime, but can't be -// used for arbitrarily nested expressions. -class DirectExpressionStep { - public: - explicit DirectExpressionStep(int64_t expr_id) : expr_id_(expr_id) {} - DirectExpressionStep() : expr_id_(-1) {} - - virtual ~DirectExpressionStep() = default; - - int64_t expr_id() const { return expr_id_; } - bool comes_from_ast() const { return expr_id_ >= 0; } - - virtual absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute) const = 0; - - // Return a type id for this node. - // - // Users must not make any assumptions about the type if the default value is - // returned. - virtual cel::NativeTypeId GetNativeTypeId() const { - return cel::NativeTypeId(); - } - - // Implementations optionally support inspecting the program tree. - virtual absl::optional> - GetDependencies() const { - return absl::nullopt; - } - - // Implementations optionally support extracting the program tree. - // - // Extract prevents the callee from functioning, and is only intended for use - // when replacing a given expression step. - virtual absl::optional>> - ExtractDependencies() { - return absl::nullopt; - }; - - protected: - int64_t expr_id_; -}; - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_DIRECT_EXPRESSION_STEP_H_ diff --git a/eval/eval/equality_steps.cc b/eval/eval/equality_steps.cc index 483b91468..6d8cf5751 100644 --- a/eval/eval/equality_steps.cc +++ b/eval/eval/equality_steps.cc @@ -13,17 +13,15 @@ // limitations under the License. #include "eval/eval/equality_steps.h" -#include -#include #include #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" #include "base/builtins.h" #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "internal/number.h" #include "internal/status_macros.h" @@ -76,36 +74,6 @@ absl::StatusOr EvaluateEquality( return negation ? BoolValue(!*is_equal) : BoolValue(*is_equal); } -class DirectEqualityStep : public DirectExpressionStep { - public: - explicit DirectEqualityStep(std::unique_ptr lhs, - std::unique_ptr rhs, - bool negation, int64_t expr_id) - : DirectExpressionStep(expr_id), - lhs_(std::move(lhs)), - rhs_(std::move(rhs)), - negation_(negation) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override { - AttributeTrail lhs_attr; - CEL_RETURN_IF_ERROR(lhs_->Evaluate(frame, result, lhs_attr)); - - Value rhs_result; - AttributeTrail rhs_attr; - CEL_RETURN_IF_ERROR(rhs_->Evaluate(frame, rhs_result, rhs_attr)); - CEL_ASSIGN_OR_RETURN( - result, EvaluateEquality(frame, result, lhs_attr, rhs_result, rhs_attr, - negation_)); - return absl::OkStatus(); - } - - private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; - bool negation_; -}; - absl::StatusOr EvaluateInMap(ExecutionFrameBase& frame, const Value& item, const MapValue& container) { @@ -193,34 +161,6 @@ absl::StatusOr EvaluateIn(ExecutionFrameBase& frame, const Value& item, frame.arena()); } -class DirectInStep : public DirectExpressionStep { - public: - explicit DirectInStep(std::unique_ptr item, - std::unique_ptr container, - int64_t expr_id) - : DirectExpressionStep(expr_id), - item_(std::move(item)), - container_(std::move(container)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override { - AttributeTrail item_attr; - CEL_RETURN_IF_ERROR(item_->Evaluate(frame, result, item_attr)); - - Value container_result; - AttributeTrail container_attr; - CEL_RETURN_IF_ERROR( - container_->Evaluate(frame, container_result, container_attr)); - CEL_ASSIGN_OR_RETURN(result, EvaluateIn(frame, result, item_attr, - container_result, container_attr)); - return absl::OkStatus(); - } - - private: - std::unique_ptr item_; - std::unique_ptr container_; -}; - } // namespace void EvaluateFastEqualStep(bool negation, ExecutionFrame& frame) { @@ -261,20 +201,4 @@ void EvaluateFastInStep(ExecutionFrame& frame) { frame.value_stack().PopAndPush(2, *std::move(result)); } -// Factory method for recursive _==_ and _!=_ Execution step -std::unique_ptr CreateDirectEqualityStep( - std::unique_ptr lhs, - std::unique_ptr rhs, bool negation, int64_t expr_id) { - return std::make_unique(std::move(lhs), std::move(rhs), - negation, expr_id); -} - -// Factory method for recursive @in Execution step -std::unique_ptr CreateDirectInStep( - std::unique_ptr item, - std::unique_ptr container, int64_t expr_id) { - return std::make_unique(std::move(item), std::move(container), - expr_id); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/equality_steps.h b/eval/eval/equality_steps.h index 141e571d1..18b87cc79 100644 --- a/eval/eval/equality_steps.h +++ b/eval/eval/equality_steps.h @@ -15,27 +15,12 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_EQUALITY_STEPS_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_EQUALITY_STEPS_H_ -#include -#include - -#include "eval/eval/direct_expression_step.h" - namespace google::api::expr::runtime { class ExecutionFrame; -// Factory method for recursive _==_/_!=_ Execution step -std::unique_ptr CreateDirectEqualityStep( - std::unique_ptr lhs, - std::unique_ptr rhs, bool negation, int64_t expr_id); - void EvaluateFastEqualStep(bool negation, ExecutionFrame& frame); -// Factory method for recursive @in Execution step -std::unique_ptr CreateDirectInStep( - std::unique_ptr item, - std::unique_ptr container, int64_t expr_id); - void EvaluateFastInStep(ExecutionFrame& frame); } // namespace google::api::expr::runtime diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index 660bebeb8..4ebfe10ef 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -27,8 +27,8 @@ #include "common/value_kind.h" #include "common/value_testing.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "internal/testing.h" #include "internal/testing_descriptor_pool.h" #include "internal/testing_message_factory.h" @@ -51,16 +51,14 @@ using ::cel::ValueKind; using ::cel::test::BoolValueIs; using ::cel::test::ValueKindIs; -class ValueStep : public ExpressionStepLogic, public DirectExpressionStep { +class ValueStep : public ExpressionStepLogic { public: ValueStep(Value value, Attribute attr) : ExpressionStepLogic(), - DirectExpressionStep(-1), value_(std::move(value)), attr_(std::move(attr)) {} explicit ValueStep(Value value) : ExpressionStepLogic(), - DirectExpressionStep(-1), value_(std::move(value)), attr_() {} @@ -68,73 +66,11 @@ class ValueStep : public ExpressionStepLogic, public DirectExpressionStep { frame->value_stack().Push(value_, attr_); } - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override { - result = value_; - attribute_trail = attr_; - return absl::OkStatus(); - } - private: Value value_; AttributeTrail attr_; }; -TEST(RecursiveTest, PartialAttrUnknown) { - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - // A little contrived for simplicity, but this is for cases where e.g. - // `msg == Msg{}` but msg.foo is unknown. - auto plan = CreateDirectEqualityStep( - std::make_unique(IntValue(1), cel::Attribute("foo")), - std::make_unique(IntValue(2)), false, -1); - - ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}), - IsOk()); - - ExecutionFrameBase frame(activation, opts, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - cel::Value result; - AttributeTrail attribute_trail; - ASSERT_THAT(plan->Evaluate(frame, result, attribute_trail), IsOk()); - - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST(RecursiveTest, PartialAttrUnknownDisabled) { - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kDisabled; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - auto plan = CreateDirectEqualityStep( - std::make_unique(IntValue(1), cel::Attribute("foo")), - std::make_unique(IntValue(2)), false, -1); - - ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( - "foo", {cel::AttributeQualifierPattern::OfString("bar")})}), - IsOk()); - ExecutionFrameBase frame(activation, opts, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - cel::Value result; - AttributeTrail attribute_trail; - ASSERT_THAT(plan->Evaluate(frame, result, attribute_trail), IsOk()); - - EXPECT_THAT(result, BoolValueIs(false)); -} - TEST(IterativeTest, PartialAttrUnknown) { cel::Activation activation; google::protobuf::Arena arena; @@ -236,44 +172,6 @@ Value MakeValue(InputType type, google::protobuf::Arena* absl_nonnull arena) { } } -TEST_P(EqualsTest, Recursive) { - const EqualsTestCase& test_case = GetParam(); - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - auto plan = CreateDirectEqualityStep( - std::make_unique(MakeValue(test_case.lhs, &arena)), - std::make_unique(MakeValue(test_case.rhs, &arena)), - test_case.negation, -1); - - ExecutionFrameBase frame(activation, opts, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - cel::Value result; - AttributeTrail attribute_trail; - ASSERT_THAT(plan->Evaluate(frame, result, attribute_trail), IsOk()); - - switch (test_case.expected_result) { - case OutputType::kBoolTrue: - EXPECT_THAT(result, BoolValueIs(true)); - break; - case OutputType::kBoolFalse: - EXPECT_THAT(result, BoolValueIs(false)); - break; - case OutputType::kError: - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); - break; - case OutputType::kUnknown: - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); - break; - } -} - TEST_P(EqualsTest, Iterative) { const EqualsTestCase& test_case = GetParam(); cel::Activation activation; @@ -419,43 +317,6 @@ struct InTestCase { class InTest : public ::testing::TestWithParam {}; -TEST_P(InTest, Recursive) { - const InTestCase& test_case = GetParam(); - cel::Activation activation; - google::protobuf::Arena arena; - cel::RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - - auto plan = CreateDirectInStep( - std::make_unique(MakeValue(test_case.lhs, &arena)), - std::make_unique(MakeValue(test_case.rhs, &arena)), -1); - - ExecutionFrameBase frame(activation, opts, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - - cel::Value result; - AttributeTrail attribute_trail; - ASSERT_THAT(plan->Evaluate(frame, result, attribute_trail), IsOk()); - - switch (test_case.expected_result) { - case OutputType::kBoolTrue: - EXPECT_THAT(result, BoolValueIs(true)); - break; - case OutputType::kBoolFalse: - EXPECT_THAT(result, BoolValueIs(false)); - break; - case OutputType::kError: - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); - break; - case OutputType::kUnknown: - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); - break; - } -} - TEST_P(InTest, Iterative) { const InTestCase& test_case = GetParam(); cel::Activation activation; diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 6d31a7b4b..4314f4e38 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -17,11 +17,9 @@ #include #include #include -#include #include #include -#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/log/absl_check.h" @@ -33,15 +31,12 @@ #include "common/value.h" #include "common/value_kind.h" #include "common/values/list_value_builder.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/comprehension_slots.h" #include "eval/eval/comprehension_step.h" #include "eval/eval/equality_steps.h" #include "eval/eval/lazy_init_step.h" #include "eval/eval/logic_step.h" -#include "internal/status_macros.h" #include "runtime/activation_interface.h" -#include "runtime/internal/errors.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" @@ -365,15 +360,4 @@ FixedJumpStepInfo* GetIfFixedJumpStep(ExpressionStep& step) { return nullptr; } -void WrappedDirectStep::Evaluate(ExecutionFrame* frame) const { - cel::Value result; - AttributeTrail attribute_trail; - if (absl::Status status = impl_->Evaluate(*frame, result, attribute_trail); - !status.ok()) { - frame->Abort(std::move(status)); - return; - } - frame->value_stack().Push(std::move(result), std::move(attribute_trail)); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 30e2c0087..d23dfd045 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -33,14 +33,12 @@ #include "absl/strings/string_view.h" #include "absl/types/span.h" #include "base/type_provider.h" -#include "common/native_type.h" #include "common/value.h" #include "common/value_kind.h" #include "common/values/list_value_builder.h" #include "eval/eval/attribute_utility.h" #include "eval/eval/comprehension_slots.h" #include "eval/eval/comprehension_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/equality_steps.h" #include "eval/eval/evaluator_stack.h" #include "eval/eval/expression_step_logic.h" @@ -423,25 +421,6 @@ class ExpressionStep { static_assert(sizeof(ExpressionStep) == 16); #endif -// Wrapper for direct steps to work with the stack machine impl. -class WrappedDirectStep : public ExpressionStepLogic { - public: - explicit WrappedDirectStep(std::unique_ptr impl, - int64_t expr_id = -1) - : impl_(std::move(impl)) {} - - void Evaluate(ExecutionFrame* frame) const override; - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - const DirectExpressionStep* wrapped() const { return impl_.get(); } - - private: - std::unique_ptr impl_; -}; - using ExecutionPath = std::vector; using ExecutionPathView = absl::Span; diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index f3d3fb93a..4160072fc 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -9,7 +9,6 @@ #include #include -#include "absl/container/inlined_vector.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" @@ -22,7 +21,6 @@ #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" @@ -303,81 +301,6 @@ class LazyResolver { bool receiver_style_; }; -template -class DirectFunctionStepImpl : public DirectExpressionStep { - public: - DirectFunctionStepImpl( - int64_t expr_id, const std::string& name, - std::vector> arg_steps, - bool receiver_style, Resolver&& resolver) - : DirectExpressionStep(expr_id), - name_(name), - arg_steps_(std::move(arg_steps)), - receiver_style_(receiver_style), - resolver_(std::forward(resolver)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& trail) const override { - absl::InlinedVector args; - absl::InlinedVector arg_trails; - - args.resize(arg_steps_.size()); - arg_trails.resize(arg_steps_.size()); - - for (size_t i = 0; i < arg_steps_.size(); i++) { - CEL_RETURN_IF_ERROR( - arg_steps_[i]->Evaluate(frame, args[i], arg_trails[i])); - } - - if (frame.unknown_processing_enabled()) { - for (size_t i = 0; i < arg_trails.size(); i++) { - if (frame.attribute_utility().CheckForUnknown(arg_trails[i], - /*use_partial=*/true)) { - args[i] = frame.attribute_utility().CreateUnknownSet( - arg_trails[i].attribute()); - } - } - } - - CEL_ASSIGN_OR_RETURN(ResolveResult resolved_function, - resolver_.Resolve(frame, args)); - - if (resolved_function.has_value() && - ShouldAcceptOverload(resolved_function->descriptor, args)) { - CEL_ASSIGN_OR_RETURN(result, - Invoke(*resolved_function, expr_id_, args, frame)); - - return absl::OkStatus(); - } - - result = NoOverloadResult(name_, args, receiver_style_, frame); - - return absl::OkStatus(); - } - - absl::optional> GetDependencies() - const override { - std::vector dependencies; - dependencies.reserve(arg_steps_.size()); - for (const auto& arg_step : arg_steps_) { - dependencies.push_back(arg_step.get()); - } - return dependencies; - } - - absl::optional>> - ExtractDependencies() override { - return std::move(arg_steps_); - } - - private: - friend Resolver; - std::string name_; - std::vector> arg_steps_; - bool receiver_style_; - Resolver resolver_; -}; - } // namespace template @@ -463,24 +386,6 @@ absl::StatusOr LazyFunctionStep::ResolveFunction( return ResolveLazy(input_args, name_, receiver_style_, providers_, frame); } -std::unique_ptr CreateDirectFunctionStep( - int64_t expr_id, const cel::CallExpr& call, - std::vector> deps, - std::vector overloads) { - return std::make_unique>( - expr_id, call.function(), std::move(deps), call.has_target(), - StaticResolver(std::move(overloads))); -} - -std::unique_ptr CreateDirectLazyFunctionStep( - int64_t expr_id, const cel::CallExpr& call, - std::vector> deps, - std::vector providers) { - return std::make_unique>( - expr_id, call.function(), std::move(deps), call.has_target(), - LazyResolver(std::move(providers), call.function(), call.has_target())); -} - std::unique_ptr CreateLazyFunctionStep( const cel::CallExpr& call_expr, int64_t expr_id, std::vector lazy_overloads) { diff --git a/eval/eval/function_step.h b/eval/eval/function_step.h index e0373d670..c5d4893da 100644 --- a/eval/eval/function_step.h +++ b/eval/eval/function_step.h @@ -14,29 +14,12 @@ #include "absl/types/span.h" #include "common/expr.h" #include "common/value.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/expression_step_logic.h" #include "runtime/function_overload_reference.h" #include "runtime/function_registry.h" namespace google::api::expr::runtime { -// Factory method for Call-based execution step where the function has been -// statically resolved from a set of eagerly functions configured in the -// CelFunctionRegistry. -std::unique_ptr CreateDirectFunctionStep( - int64_t expr_id, const cel::CallExpr& call, - std::vector> deps, - std::vector overloads); - -// Factory method for Call-based execution step where the function has been -// statically resolved from a set of lazy functions configured in the -// CelFunctionRegistry. -std::unique_ptr CreateDirectLazyFunctionStep( - int64_t expr_id, const cel::CallExpr& call, - std::vector> deps, - std::vector providers); - class LazyFunctionStep; class EagerFunctionStep; class ExecutionFrameBase; diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index c6de63d42..9cb338af2 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -8,22 +8,17 @@ #include #include "absl/status/status.h" -#include "absl/status/status_macros.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" -#include "base/builtins.h" #include "base/type_provider.h" #include "common/constant.h" #include "common/expr.h" #include "common/kind.h" #include "common/value.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/internal/interop.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" @@ -36,11 +31,8 @@ #include "eval/public/testing/matchers.h" #include "eval/testutil/test_message.pb.h" #include "internal/testing.h" -#include "runtime/function_overload_reference.h" -#include "runtime/function_registry.h" #include "runtime/internal/runtime_env_testing.h" #include "runtime/runtime_options.h" -#include "runtime/standard_functions.h" #include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -56,7 +48,6 @@ using ::cel::TypeProvider; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::testing::Eq; using ::testing::Not; -using ::testing::Truly; int GetExprId() { static int id = 0; @@ -213,20 +204,6 @@ std::vector ArgumentMatcher(const CallExpr& call) { : call.args().size()); } -std::unique_ptr CreateExpressionImpl( - const cel::RuntimeOptions& options, - std::unique_ptr expr) { - ExecutionPath path; - path.push_back(ExpressionStep::MakeGenericStep( - std::make_unique(std::move(expr)))); - - auto env = NewTestingRuntimeEnv(); - return std::make_unique( - env, - FlatExpression(std::move(path), /*comprehension_slot_count=*/0, - env->type_registry.GetComposedTypeProvider(), options)); -} - absl::StatusOr MakeTestFunctionStep( const CallExpr& call, const CelFunctionRegistry& registry) { auto argument_matcher = ArgumentMatcher(call); @@ -1045,175 +1022,5 @@ TEST(FunctionStepStrictnessTest, IfFunctionNonStrictAndGivenUnknownInvokesIt) { ASSERT_THAT(value, test::IsCelInt64(Eq(0))); } -class DirectFunctionStepTest : public testing::Test { - public: - DirectFunctionStepTest() = default; - - void SetUp() override { - ASSERT_OK(cel::RegisterStandardFunctions(registry_, options_)); - } - - std::vector GetOverloads( - absl::string_view name, int64_t arguments_size) { - std::vector matcher; - matcher.resize(arguments_size, cel::Kind::kAny); - return registry_.FindStaticOverloads(name, false, matcher); - } - - // Helper for shorthand constructing direct expr deps. - // - // Works around copies in init-list construction. - std::vector> MakeDeps( - std::unique_ptr dep, - std::unique_ptr dep2) { - std::vector> result; - result.reserve(2); - result.push_back(std::move(dep)); - result.push_back(std::move(dep2)); - return result; - }; - - protected: - cel::FunctionRegistry registry_; - cel::RuntimeOptions options_; - google::protobuf::Arena arena_; -}; - -TEST_F(DirectFunctionStepTest, SimpleCall) { - cel::IntValue(1); - - CallExpr call; - call.set_function(cel::builtin::kAdd); - call.mutable_args().emplace_back(); - call.mutable_args().emplace_back(); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(cel::IntValue(1))); - deps.push_back(CreateConstValueDirectStep(cel::IntValue(1))); - - auto expr = CreateDirectFunctionStep(-1, call, std::move(deps), - GetOverloads(cel::builtin::kAdd, 2)); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, test::IsCelInt64(2)); -} - -TEST_F(DirectFunctionStepTest, RecursiveCall) { - cel::IntValue(1); - - CallExpr call; - call.set_function(cel::builtin::kAdd); - call.mutable_args().emplace_back(); - call.mutable_args().emplace_back(); - - auto overloads = GetOverloads(cel::builtin::kAdd, 2); - - auto MakeLeaf = [&]() { - return CreateDirectFunctionStep( - -1, call, - MakeDeps(CreateConstValueDirectStep(cel::IntValue(1)), - CreateConstValueDirectStep(cel::IntValue(1))), - overloads); - }; - - auto expr = CreateDirectFunctionStep( - -1, call, - MakeDeps(CreateDirectFunctionStep( - -1, call, MakeDeps(MakeLeaf(), MakeLeaf()), overloads), - CreateDirectFunctionStep( - -1, call, MakeDeps(MakeLeaf(), MakeLeaf()), overloads)), - overloads); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, test::IsCelInt64(8)); -} - -TEST_F(DirectFunctionStepTest, ErrorHandlingCall) { - cel::IntValue(1); - - CallExpr add_call; - add_call.set_function(cel::builtin::kAdd); - add_call.mutable_args().emplace_back(); - add_call.mutable_args().emplace_back(); - - CallExpr div_call; - div_call.set_function(cel::builtin::kDivide); - div_call.mutable_args().emplace_back(); - div_call.mutable_args().emplace_back(); - - auto add_overloads = GetOverloads(cel::builtin::kAdd, 2); - auto div_overloads = GetOverloads(cel::builtin::kDivide, 2); - - auto error_expr = CreateDirectFunctionStep( - -1, div_call, - MakeDeps(CreateConstValueDirectStep(cel::IntValue(1)), - CreateConstValueDirectStep(cel::IntValue(0))), - div_overloads); - - auto expr = CreateDirectFunctionStep( - -1, add_call, - MakeDeps(std::move(error_expr), - CreateConstValueDirectStep(cel::IntValue(1))), - add_overloads); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, - test::IsCelError(StatusIs(absl::StatusCode::kInvalidArgument, - testing::HasSubstr("divide by zero")))); -} - -TEST_F(DirectFunctionStepTest, NoOverload) { - cel::IntValue(1); - - CallExpr call; - call.set_function(cel::builtin::kAdd); - call.mutable_args().emplace_back(); - call.mutable_args().emplace_back(); - - std::vector> deps; - deps.push_back(CreateConstValueDirectStep(cel::IntValue(1))); - deps.push_back(CreateConstValueDirectStep(cel::StringValue::WrapUnsafe("2"))); - - auto expr = CreateDirectFunctionStep(-1, call, std::move(deps), - GetOverloads(cel::builtin::kAdd, 2)); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, Truly(CheckNoMatchingOverloadError)); -} - -TEST_F(DirectFunctionStepTest, NoOverload0Args) { - cel::IntValue(1); - - CallExpr call; - call.set_function(cel::builtin::kAdd); - - std::vector> deps; - auto expr = CreateDirectFunctionStep(-1, call, std::move(deps), - GetOverloads(cel::builtin::kAdd, 2)); - - auto plan = CreateExpressionImpl(options_, std::move(expr)); - - Activation activation; - ASSERT_OK_AND_ASSIGN(auto value, plan->Evaluate(activation, &arena_)); - - EXPECT_THAT(value, Truly(CheckNoMatchingOverloadError)); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index 088e39bbe..0bfbb9211 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -1,20 +1,13 @@ #include "eval/eval/ident_step.h" -#include -#include -#include #include #include -#include "absl/base/nullability.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" @@ -63,54 +56,6 @@ absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, return absl::OkStatus(); } -absl::StatusOr LookupSlot( - absl::string_view name, size_t slot_index, ExecutionFrameBase& frame) { - ComprehensionSlots::Slot* slot = frame.comprehension_slots().Get(slot_index); - if (!slot->Has()) { - return absl::InternalError( - absl::StrCat("Comprehension variable accessed out of scope: ", name)); - } - return slot; -} - -class DirectIdentStep : public DirectExpressionStep { - public: - DirectIdentStep(absl::string_view name, int64_t expr_id) - : DirectExpressionStep(expr_id), name_(name) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - return LookupIdent(name_, frame, result, attribute); - } - - private: - std::string name_; -}; - -class DirectSlotStep : public DirectExpressionStep { - public: - DirectSlotStep(absl::string_view name, size_t slot_index, int64_t expr_id) - : DirectExpressionStep(expr_id), - name_(std::move(name)), - slot_index_(slot_index) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - CEL_ASSIGN_OR_RETURN(const ComprehensionSlots::Slot* slot, - LookupSlot(name_, slot_index_, frame)); - - if (frame.attribute_tracking_enabled()) { - attribute = slot->attribute(); - } - result = slot->value(); - return absl::OkStatus(); - } - - private: - std::string name_; - size_t slot_index_; -}; - } // namespace void EvaluateIdentifierStep(absl::string_view identifier, @@ -125,14 +70,4 @@ void EvaluateIdentifierStep(absl::string_view identifier, } } -std::unique_ptr CreateDirectIdentStep( - absl::string_view identifier, int64_t expr_id) { - return std::make_unique(identifier, expr_id); -} - -std::unique_ptr CreateDirectSlotIdentStep( - absl::string_view identifier, size_t slot_index, int64_t expr_id) { - return std::make_unique(identifier, slot_index, expr_id); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/ident_step.h b/eval/eval/ident_step.h index 9bafa77fe..2ce50c58e 100644 --- a/eval/eval/ident_step.h +++ b/eval/eval/ident_step.h @@ -1,24 +1,12 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_IDENT_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_IDENT_STEP_H_ -#include -#include -#include - #include "absl/strings/string_view.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { class ExecutionFrame; -std::unique_ptr CreateDirectIdentStep( - absl::string_view identifier, int64_t expr_id); - -std::unique_ptr CreateDirectSlotIdentStep( - absl::string_view identifier, size_t slot_index, int64_t expr_id); - void EvaluateIdentifierStep(absl::string_view identifier, ExecutionFrame& frame); diff --git a/eval/eval/ident_step_test.cc b/eval/eval/ident_step_test.cc index 91b2cb986..49ef870d6 100644 --- a/eval/eval/ident_step_test.cc +++ b/eval/eval/ident_step_test.cc @@ -8,19 +8,13 @@ #include "absl/status/status.h" #include "base/type_provider.h" #include "common/memory.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" #include "eval/eval/evaluator_core.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -29,19 +23,12 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::absl_testing::StatusIs; -using ::cel::ErrorValue; -using ::cel::IntValue; using ::cel::MemoryManagerRef; using ::cel::RuntimeOptions; using ::cel::TypeProvider; -using ::cel::UnknownValue; -using ::cel::Value; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::google::protobuf::Arena; using ::testing::Eq; -using ::testing::HasSubstr; -using ::testing::SizeIs; TEST(IdentStepTest, TestIdentStep) { ExecutionPath path; @@ -202,107 +189,6 @@ TEST(IdentStepTest, TestIdentStepUnknownAttribute) { ASSERT_TRUE(result.IsUnknownSet()); } -TEST(DirectIdentStepTest, Basic) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - cel::Activation activation; - RuntimeOptions options; - - activation.InsertOrAssignValue("var1", IntValue(42)); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - Value result; - AttributeTrail trail; - - auto step = CreateDirectIdentStep("var1", -1); - - ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); - - ASSERT_TRUE(result.IsInt()); - EXPECT_THAT(result.GetInt().NativeValue(), Eq(42)); -} - -TEST(DirectIdentStepTest, UnknownAttribute) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - activation.InsertOrAssignValue("var1", IntValue(42)); - ASSERT_THAT( - activation.SetUnknownPatterns({CreateCelAttributePattern("var1", {})}), - IsOk()); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - Value result; - AttributeTrail trail; - - auto step = CreateDirectIdentStep("var1", -1); - - ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); - - ASSERT_TRUE(result.IsUnknown()); - EXPECT_THAT(result.GetUnknown().ToAttributeSet(), SizeIs(1)); -} - -TEST(DirectIdentStepTest, MissingAttribute) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - cel::Activation activation; - RuntimeOptions options; - options.enable_missing_attribute_errors = true; - - activation.InsertOrAssignValue("var1", IntValue(42)); - ASSERT_THAT( - activation.SetMissingPatterns({CreateCelAttributePattern("var1", {})}), - IsOk()); - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - Value result; - AttributeTrail trail; - - auto step = CreateDirectIdentStep("var1", -1); - - ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("var1"))); -} - -TEST(DirectIdentStepTest, NotFound) { - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - cel::Activation activation; - RuntimeOptions options; - - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - Value result; - AttributeTrail trail; - - auto step = CreateDirectIdentStep("var1", -1); - - ASSERT_THAT(step->Evaluate(frame, result, trail), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kUnknown, - HasSubstr("\"var1\" found in Activation"))); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step.cc b/eval/eval/lazy_init_step.cc index cdb1f9666..03404d048 100644 --- a/eval/eval/lazy_init_step.cc +++ b/eval/eval/lazy_init_step.cc @@ -15,103 +15,14 @@ #include "eval/eval/lazy_init_step.h" #include -#include -#include -#include #include "cel/expr/value.pb.h" -#include "absl/base/nullability.h" #include "absl/status/status.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "internal/status_macros.h" namespace google::api::expr::runtime { -namespace { - -using ::cel::Value; - -class DirectLazyInitStep final : public DirectExpressionStep { - public: - DirectLazyInitStep(size_t slot_index, - const DirectExpressionStep* subexpression, int64_t expr_id) - : DirectExpressionStep(expr_id), - slot_index_(slot_index), - subexpression_(subexpression) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - ComprehensionSlot* slot = frame.comprehension_slots().Get(slot_index_); - if (slot->Has()) { - result = slot->value(); - attribute = slot->attribute(); - } else { - CEL_RETURN_IF_ERROR(subexpression_->Evaluate(frame, result, attribute)); - slot->Set(result, attribute); - } - return absl::OkStatus(); - } - - private: - const size_t slot_index_; - const DirectExpressionStep* absl_nonnull const subexpression_; -}; - -class BindStep : public DirectExpressionStep { - public: - BindStep(size_t slot_index, - std::unique_ptr subexpression, int64_t expr_id) - : DirectExpressionStep(expr_id), - slot_index_(slot_index), - subexpression_(std::move(subexpression)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - CEL_RETURN_IF_ERROR(subexpression_->Evaluate(frame, result, attribute)); - - frame.comprehension_slots().ClearSlot(slot_index_); - - return absl::OkStatus(); - } - - private: - size_t slot_index_; - std::unique_ptr subexpression_; -}; - -class BlockStep : public DirectExpressionStep { - public: - BlockStep(size_t slot_index, size_t slot_count, - std::unique_ptr subexpression, - int64_t expr_id) - : DirectExpressionStep(expr_id), - slot_index_(slot_index), - slot_count_(slot_count), - subexpression_(std::move(subexpression)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - CEL_RETURN_IF_ERROR(subexpression_->Evaluate(frame, result, attribute)); - - for (size_t i = 0; i < slot_count_; ++i) { - frame.comprehension_slots().ClearSlot(slot_index_ + i); - } - - return absl::OkStatus(); - } - - private: - size_t slot_index_; - size_t slot_count_; - std::unique_ptr subexpression_; -}; - -} // namespace - void EvaluateLazyInitStep(const LazyInitStepInfo& step, ExecutionFrame& frame) { ComprehensionSlot* slot = frame.comprehension_slots().Get(step.slot_index); if (slot->Has()) { @@ -138,24 +49,4 @@ void EvaluateClearSlotStep(const ClearSlotStepInfo& step, } } -std::unique_ptr CreateDirectBindStep( - size_t slot_index, std::unique_ptr expression, - int64_t expr_id) { - return std::make_unique(slot_index, std::move(expression), expr_id); -} - -std::unique_ptr CreateDirectBlockStep( - size_t slot_index, size_t slot_count, - std::unique_ptr expression, int64_t expr_id) { - return std::make_unique(slot_index, slot_count, - std::move(expression), expr_id); -} - -std::unique_ptr CreateDirectLazyInitStep( - size_t slot_index, const DirectExpressionStep* absl_nonnull subexpression, - int64_t expr_id) { - return std::make_unique(slot_index, subexpression, - expr_id); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/lazy_init_step.h b/eval/eval/lazy_init_step.h index c73c80ee1..a1838c9b3 100644 --- a/eval/eval/lazy_init_step.h +++ b/eval/eval/lazy_init_step.h @@ -38,32 +38,11 @@ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_LAZY_INIT_STEP_H_ #include -#include -#include - -#include "absl/base/nullability.h" -#include "eval/eval/direct_expression_step.h" namespace google::api::expr::runtime { class ExecutionFrame; -// Creates a step representing a Bind expression. -std::unique_ptr CreateDirectBindStep( - size_t slot_index, std::unique_ptr expression, - int64_t expr_id); - -// Creates a step representing a cel.@block expression. -std::unique_ptr CreateDirectBlockStep( - size_t slot_index, size_t slot_count, - std::unique_ptr expression, int64_t expr_id); - -// Creates a direct step representing accessing a lazily evaluated alias from -// a bind or block. -std::unique_ptr CreateDirectLazyInitStep( - size_t slot_index, const DirectExpressionStep* absl_nonnull subexpression, - int64_t expr_id); - struct LazyInitStepInfo { size_t slot_index : 32; size_t subexpression_index : 32; diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index d7bab5f8c..3be16e527 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -1,8 +1,6 @@ #include "eval/eval/logic_step.h" #include -#include -#include #include #include @@ -13,10 +11,8 @@ #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/internal/errors.h" -#include "internal/status_macros.h" #include "runtime/internal/errors.h" namespace google::api::expr::runtime { @@ -30,247 +26,6 @@ using ::cel::Value; using ::cel::ValueKind; using ::cel::runtime_internal::CreateNoMatchingOverloadError; -// Shared logic for the fall through case (we didn't see the shortcircuit -// value). -absl::Status ReturnLogicResult(ExecutionFrameBase& frame, BoolLogicKind op_type, - Value& lhs_result, Value& rhs_result, - AttributeTrail& attribute_trail, - AttributeTrail& rhs_attr) { - ValueKind lhs_kind = lhs_result.kind(); - ValueKind rhs_kind = rhs_result.kind(); - - if (frame.unknown_processing_enabled()) { - if (lhs_kind == ValueKind::kUnknown && rhs_kind == ValueKind::kUnknown) { - lhs_result = frame.attribute_utility().MergeUnknownValues( - lhs_result.GetUnknown(), rhs_result.GetUnknown()); - // Clear attribute trail so this doesn't get re-identified as a new - // unknown and reset the accumulated attributes. - attribute_trail = AttributeTrail(); - return absl::OkStatus(); - } else if (lhs_kind == ValueKind::kUnknown) { - return absl::OkStatus(); - } else if (rhs_kind == ValueKind::kUnknown) { - lhs_result = std::move(rhs_result); - attribute_trail = std::move(rhs_attr); - return absl::OkStatus(); - } - } - - if (lhs_kind == ValueKind::kError) { - return absl::OkStatus(); - } else if (rhs_kind == ValueKind::kError) { - lhs_result = std::move(rhs_result); - attribute_trail = std::move(rhs_attr); - return absl::OkStatus(); - } - - if (lhs_kind == ValueKind::kBool && rhs_kind == ValueKind::kBool) { - return absl::OkStatus(); - } - - // Otherwise, add a no overload error. - attribute_trail = AttributeTrail(); - lhs_result = cel::ErrorValue::From( - CreateNoMatchingOverloadError(op_type == BoolLogicKind::kOr - ? cel::builtin::kOr - : cel::builtin::kAnd), - frame.arena()); - return absl::OkStatus(); -} - -class ExhaustiveDirectLogicStep : public DirectExpressionStep { - public: - explicit ExhaustiveDirectLogicStep(std::unique_ptr lhs, - std::unique_ptr rhs, - BoolLogicKind op_type, int64_t expr_id) - : DirectExpressionStep(expr_id), - lhs_(std::move(lhs)), - rhs_(std::move(rhs)), - op_type_(op_type) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; - BoolLogicKind op_type_; -}; - -absl::Status ExhaustiveDirectLogicStep::Evaluate( - ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute_trail) const { - CEL_RETURN_IF_ERROR(lhs_->Evaluate(frame, result, attribute_trail)); - ValueKind lhs_kind = result.kind(); - - Value rhs_result; - AttributeTrail rhs_attr; - CEL_RETURN_IF_ERROR(rhs_->Evaluate(frame, rhs_result, attribute_trail)); - - ValueKind rhs_kind = rhs_result.kind(); - if (lhs_kind == ValueKind::kBool) { - bool lhs_bool = result.GetBool().NativeValue(); - if ((op_type_ == BoolLogicKind::kOr && lhs_bool) || - (op_type_ == BoolLogicKind::kAnd && !lhs_bool)) { - return absl::OkStatus(); - } - } - - if (rhs_kind == ValueKind::kBool) { - bool rhs_bool = rhs_result.GetBool().NativeValue(); - if ((op_type_ == BoolLogicKind::kOr && rhs_bool) || - (op_type_ == BoolLogicKind::kAnd && !rhs_bool)) { - result = std::move(rhs_result); - attribute_trail = std::move(rhs_attr); - return absl::OkStatus(); - } - } - - return ReturnLogicResult(frame, op_type_, result, rhs_result, attribute_trail, - rhs_attr); -} - -class DirectLogicStep : public DirectExpressionStep { - public: - explicit DirectLogicStep(std::unique_ptr lhs, - std::unique_ptr rhs, - BoolLogicKind op_type, int64_t expr_id) - : DirectExpressionStep(expr_id), - lhs_(std::move(lhs)), - rhs_(std::move(rhs)), - op_type_(op_type) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; - BoolLogicKind op_type_; -}; - -absl::Status DirectLogicStep::Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - CEL_RETURN_IF_ERROR(lhs_->Evaluate(frame, result, attribute_trail)); - ValueKind lhs_kind = result.kind(); - if (lhs_kind == ValueKind::kBool) { - bool lhs_bool = result.GetBool().NativeValue(); - if ((op_type_ == BoolLogicKind::kOr && lhs_bool) || - (op_type_ == BoolLogicKind::kAnd && !lhs_bool)) { - return absl::OkStatus(); - } - } - - Value rhs_result; - AttributeTrail rhs_attr; - - CEL_RETURN_IF_ERROR(rhs_->Evaluate(frame, rhs_result, attribute_trail)); - - ValueKind rhs_kind = rhs_result.kind(); - - if (rhs_kind == ValueKind::kBool) { - bool rhs_bool = rhs_result.GetBool().NativeValue(); - if ((op_type_ == BoolLogicKind::kOr && rhs_bool) || - (op_type_ == BoolLogicKind::kAnd && !rhs_bool)) { - result = std::move(rhs_result); - attribute_trail = std::move(rhs_attr); - return absl::OkStatus(); - } - } - - return ReturnLogicResult(frame, op_type_, result, rhs_result, attribute_trail, - rhs_attr); -} - -std::unique_ptr CreateDirectLogicStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - BoolLogicKind op_type, bool shortcircuiting) { - if (shortcircuiting) { - return std::make_unique(std::move(lhs), std::move(rhs), - op_type, expr_id); - } else { - return std::make_unique( - std::move(lhs), std::move(rhs), op_type, expr_id); - } -} - -class DirectNotStep : public DirectExpressionStep { - public: - explicit DirectNotStep(std::unique_ptr operand, - int64_t expr_id) - : DirectExpressionStep(expr_id), operand_(std::move(operand)) {} - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::unique_ptr operand_; -}; - -absl::Status DirectNotStep::Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute_trail)); - - if (frame.unknown_processing_enabled()) { - if (frame.attribute_utility().CheckForUnknownPartial(attribute_trail)) { - result = frame.attribute_utility().CreateUnknownSet( - attribute_trail.attribute()); - return absl::OkStatus(); - } - } - - switch (result.kind()) { - case ValueKind::kBool: - result = BoolValue{!result.GetBool().NativeValue()}; - break; - case ValueKind::kUnknown: - case ValueKind::kError: - // just forward. - break; - default: - result = cel::ErrorValue::From( - CreateNoMatchingOverloadError(cel::builtin::kNot), frame.arena()); - break; - } - - return absl::OkStatus(); -} - - -class DirectNotStrictlyFalseStep : public DirectExpressionStep { - public: - explicit DirectNotStrictlyFalseStep( - std::unique_ptr operand, int64_t expr_id) - : DirectExpressionStep(expr_id), operand_(std::move(operand)) {} - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const override; - - private: - std::unique_ptr operand_; -}; - -absl::Status DirectNotStrictlyFalseStep::Evaluate( - ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute_trail) const { - CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute_trail)); - - switch (result.kind()) { - case ValueKind::kBool: - // just forward. - break; - case ValueKind::kUnknown: - case ValueKind::kError: - result = BoolValue(true); - break; - default: - result = cel::ErrorValue::From( - CreateNoMatchingOverloadError(cel::builtin::kNot), frame.arena()); - break; - } - - return absl::OkStatus(); -} - } // namespace void EvaluateNotStep(ExecutionFrame& frame) { @@ -388,33 +143,4 @@ void EvaluateBoolLogicStep(BoolLogicKind kind, size_t num_args, frame.value_stack().PopAndPush(num_args, std::move(result)); } -std::unique_ptr CreateDirectAndStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - bool shortcircuiting) { - return CreateDirectLogicStep(std::move(lhs), std::move(rhs), expr_id, - BoolLogicKind::kAnd, shortcircuiting); -} - -std::unique_ptr CreateDirectOrStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - bool shortcircuiting) { - return CreateDirectLogicStep(std::move(lhs), std::move(rhs), expr_id, - BoolLogicKind::kOr, shortcircuiting); -} - -// Factory method for recursive logical not "!" Execution step -std::unique_ptr CreateDirectNotStep( - std::unique_ptr operand, int64_t expr_id) { - return std::make_unique(std::move(operand), expr_id); -} - -// Factory method for recursive logical "@not_strictly_false" Execution step. -std::unique_ptr CreateDirectNotStrictlyFalseStep( - std::unique_ptr operand, int64_t expr_id) { - return std::make_unique(std::move(operand), - expr_id); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/logic_step.h b/eval/eval/logic_step.h index 401eaf2ca..fbcfb59ed 100644 --- a/eval/eval/logic_step.h +++ b/eval/eval/logic_step.h @@ -2,10 +2,6 @@ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_LOGIC_STEP_H_ #include -#include -#include - -#include "eval/eval/direct_expression_step.h" namespace google::api::expr::runtime { @@ -23,26 +19,6 @@ enum class BoolLogicKind { void EvaluateBoolLogicStep(BoolLogicKind kind, size_t num_args, ExecutionFrame& frame); -// Factory method for "And" Execution step -std::unique_ptr CreateDirectAndStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - bool shortcircuiting); - -// Factory method for "Or" Execution step -std::unique_ptr CreateDirectOrStep( - std::unique_ptr lhs, - std::unique_ptr rhs, int64_t expr_id, - bool shortcircuiting); - -// Factory method for recursive logical not "!" Execution step -std::unique_ptr CreateDirectNotStep( - std::unique_ptr operand, int64_t expr_id); - -// Factory method for recursive logical "@not_strictly_false" Execution step. -std::unique_ptr CreateDirectNotStrictlyFalseStep( - std::unique_ptr operand, int64_t expr_id); - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_LOGIC_STEP_H_ diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index 04b9fd127..e82c4b7cf 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -2,27 +2,19 @@ #include #include -#include #include #include #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" #include "common/expr.h" -#include "common/unknown.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" @@ -30,12 +22,8 @@ #include "eval/public/unknown_set.h" #include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -46,12 +34,8 @@ namespace { using ::absl_testing::IsOk; using ::cel::Attribute; using ::cel::AttributeSet; -using ::cel::BoolValue; using ::cel::Expr; -using ::cel::IntValue; using ::cel::TypeProvider; -using ::cel::UnknownValue; -using ::cel::Value; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::google::protobuf::Arena; @@ -318,369 +302,6 @@ TEST_F(LogicStepTest, TestOrLogicUnknownHandling) { INSTANTIATE_TEST_SUITE_P(LogicStepTest, LogicStepTest, testing::Bool()); -enum class BinaryOp { kAnd, kOr }; -enum class UnaryOp { kNot, kNotStrictlyFalse }; - -enum class OpArg { - kTrue, - kFalse, - kUnknown, - kError, - // Arbitrary incorrect type - kInt -}; - -enum class OpResult { - kTrue, - kFalse, - kUnknown, - kError, -}; - -struct BinaryTestCase { - std::string name; - BinaryOp op; - OpArg arg0; - OpArg arg1; - OpResult result; -}; - -UnknownValue MakeUnknownValue(std::string attr) { - std::vector attrs; - attrs.push_back(Attribute(std::move(attr))); - return cel::common_internal::MakeUnknownValue( - cel::Unknown(AttributeSet(attrs))); -} - -std::unique_ptr MakeArgStep(OpArg arg, - absl::string_view name, - google::protobuf::Arena* arena) { - switch (arg) { - case OpArg::kTrue: - return CreateConstValueDirectStep(BoolValue(true)); - case OpArg::kFalse: - return CreateConstValueDirectStep(BoolValue(false)); - case OpArg::kUnknown: - return CreateConstValueDirectStep(MakeUnknownValue(std::string(name))); - case OpArg::kError: - return CreateConstValueDirectStep( - cel::ErrorValue::From(absl::InternalError(name), arena)); - case OpArg::kInt: - return CreateConstValueDirectStep(IntValue(42)); - } -}; - -class DirectBinaryLogicStepTest - : public testing::TestWithParam> { - public: - DirectBinaryLogicStepTest() = default; - - bool ShortcircuitingEnabled() { return std::get<0>(GetParam()); } - const BinaryTestCase& GetTestCase() { return std::get<1>(GetParam()); } - - protected: - Arena arena_; -}; - -TEST_P(DirectBinaryLogicStepTest, TestCases) { - const BinaryTestCase& test_case = GetTestCase(); - - std::unique_ptr lhs = - MakeArgStep(test_case.arg0, "lhs", &arena_); - std::unique_ptr rhs = - MakeArgStep(test_case.arg1, "rhs", &arena_); - - std::unique_ptr op = - (test_case.op == BinaryOp::kAnd) - ? CreateDirectAndStep(std::move(lhs), std::move(rhs), -1, - ShortcircuitingEnabled()) - : CreateDirectOrStep(std::move(lhs), std::move(rhs), -1, - ShortcircuitingEnabled()); - - cel::Activation activation; - cel::RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value value; - AttributeTrail attr; - ASSERT_THAT(op->Evaluate(frame, value, attr), IsOk()); - - switch (test_case.result) { - case OpResult::kTrue: - ASSERT_TRUE(value.IsBool()); - EXPECT_TRUE(value.GetBool().NativeValue()); - break; - case OpResult::kFalse: - ASSERT_TRUE(value.IsBool()); - EXPECT_FALSE(value.GetBool().NativeValue()); - break; - case OpResult::kUnknown: - EXPECT_TRUE(value.IsUnknown()); - break; - case OpResult::kError: - EXPECT_TRUE(value.IsError()); - break; - } -} - -INSTANTIATE_TEST_SUITE_P( - DirectBinaryLogicStepTest, DirectBinaryLogicStepTest, - testing::Combine(testing::Bool(), - testing::ValuesIn>({ - { - "AndFalseFalse", - BinaryOp::kAnd, - OpArg::kFalse, - OpArg::kFalse, - OpResult::kFalse, - }, - { - "AndFalseTrue", - BinaryOp::kAnd, - OpArg::kFalse, - OpArg::kTrue, - OpResult::kFalse, - }, - { - "AndTrueFalse", - BinaryOp::kAnd, - OpArg::kTrue, - OpArg::kFalse, - OpResult::kFalse, - }, - { - "AndTrueTrue", - BinaryOp::kAnd, - OpArg::kTrue, - OpArg::kTrue, - OpResult::kTrue, - }, - - { - "AndTrueError", - BinaryOp::kAnd, - OpArg::kTrue, - OpArg::kError, - OpResult::kError, - }, - { - "AndErrorTrue", - BinaryOp::kAnd, - OpArg::kError, - OpArg::kTrue, - OpResult::kError, - }, - { - "AndFalseError", - BinaryOp::kAnd, - OpArg::kFalse, - OpArg::kError, - OpResult::kFalse, - }, - { - "AndErrorFalse", - BinaryOp::kAnd, - OpArg::kError, - OpArg::kFalse, - OpResult::kFalse, - }, - { - "AndErrorError", - BinaryOp::kAnd, - OpArg::kError, - OpArg::kError, - OpResult::kError, - }, - - { - "AndTrueUnknown", - BinaryOp::kAnd, - OpArg::kTrue, - OpArg::kUnknown, - OpResult::kUnknown, - }, - { - "AndUnknownTrue", - BinaryOp::kAnd, - OpArg::kUnknown, - OpArg::kTrue, - OpResult::kUnknown, - }, - { - "AndFalseUnknown", - BinaryOp::kAnd, - OpArg::kFalse, - OpArg::kUnknown, - OpResult::kFalse, - }, - { - "AndUnknownFalse", - BinaryOp::kAnd, - OpArg::kUnknown, - OpArg::kFalse, - OpResult::kFalse, - }, - { - "AndUnknownUnknown", - BinaryOp::kAnd, - OpArg::kUnknown, - OpArg::kUnknown, - OpResult::kUnknown, - }, - { - "AndUnknownError", - BinaryOp::kAnd, - OpArg::kUnknown, - OpArg::kError, - OpResult::kUnknown, - }, - { - "AndErrorUnknown", - BinaryOp::kAnd, - OpArg::kError, - OpArg::kUnknown, - OpResult::kUnknown, - }, - // Or cases are simplified since the logic generalizes - // and is covered by and cases. - })), - [](const testing::TestParamInfo& info) - -> std::string { - bool shortcircuiting_enabled = std::get<0>(info.param); - absl::string_view name = std::get<1>(info.param).name; - return absl::StrCat( - name, (shortcircuiting_enabled ? "ShortcircuitingEnabled" : "")); - }); - -struct UnaryTestCase { - std::string name; - UnaryOp op; - OpArg arg; - OpResult result; -}; - -class DirectUnaryLogicStepTest : public testing::TestWithParam { - public: - DirectUnaryLogicStepTest() = default; - - const UnaryTestCase& GetTestCase() { return GetParam(); } - - protected: - Arena arena_; -}; - -TEST_P(DirectUnaryLogicStepTest, TestCases) { - const UnaryTestCase& test_case = GetTestCase(); - - std::unique_ptr arg = - MakeArgStep(test_case.arg, "arg", &arena_); - - std::unique_ptr op = - (test_case.op == UnaryOp::kNot) - ? CreateDirectNotStep(std::move(arg), -1) - : CreateDirectNotStrictlyFalseStep(std::move(arg), -1); - - cel::Activation activation; - cel::RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - ExecutionFrameBase frame(activation, options, type_provider, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value value; - AttributeTrail attr; - ASSERT_THAT(op->Evaluate(frame, value, attr), IsOk()); - - switch (test_case.result) { - case OpResult::kTrue: - ASSERT_TRUE(value.IsBool()); - EXPECT_TRUE(value.GetBool().NativeValue()); - break; - case OpResult::kFalse: - ASSERT_TRUE(value.IsBool()); - EXPECT_FALSE(value.GetBool().NativeValue()); - break; - case OpResult::kUnknown: - EXPECT_TRUE(value.IsUnknown()); - break; - case OpResult::kError: - EXPECT_TRUE(value.IsError()); - break; - } -} - -INSTANTIATE_TEST_SUITE_P( - DirectUnaryLogicStepTest, DirectUnaryLogicStepTest, - testing::ValuesIn>( - {UnaryTestCase{"NotTrue", UnaryOp::kNot, OpArg::kTrue, - OpResult::kFalse}, - UnaryTestCase{"NotError", UnaryOp::kNot, OpArg::kError, - OpResult::kError}, - UnaryTestCase{"NotUnknown", UnaryOp::kNot, OpArg::kUnknown, - OpResult::kUnknown}, - UnaryTestCase{"NotInt", UnaryOp::kNot, OpArg::kInt, OpResult::kError}, - UnaryTestCase{"NotFalse", UnaryOp::kNot, OpArg::kFalse, - OpResult::kTrue}, - UnaryTestCase{"NotStrictlyFalseTrue", UnaryOp::kNotStrictlyFalse, - OpArg::kTrue, OpResult::kTrue}, - UnaryTestCase{"NotStrictlyFalseError", UnaryOp::kNotStrictlyFalse, - OpArg::kError, OpResult::kTrue}, - UnaryTestCase{"NotStrictlyFalseUnknown", UnaryOp::kNotStrictlyFalse, - OpArg::kUnknown, OpResult::kTrue}, - UnaryTestCase{"NotStrictlyFalseInt", UnaryOp::kNotStrictlyFalse, - OpArg::kInt, OpResult::kError}, - UnaryTestCase{"NotStrictlyFalseFalse", UnaryOp::kNotStrictlyFalse, - OpArg::kFalse, OpResult::kFalse}}), - [](const testing::TestParamInfo& info) - -> std::string { return info.param.name; }); - -TEST(UnaryLogicStepTest, BooleanNot) { - ExecutionPath path; - path.push_back(ExpressionStep::MakeConstant(cel::BoolValue(true))); - path.push_back(ExpressionStep::MakeBooleanNotStep()); - - google::protobuf::Arena arena; - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - FlatExpressionEvaluatorState state( - 2, 0, type_provider, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - cel::Activation activation; - cel::RuntimeOptions options; - ExecutionFrame frame(path, activation, options, state); - ASSERT_OK_AND_ASSIGN(cel::Value value, frame.Evaluate()); - ASSERT_TRUE(value.IsBool()); - EXPECT_FALSE(value.GetBool().NativeValue()); -} - -TEST(UnaryLogicStepTest, NotStrictlyFalse) { - google::protobuf::Arena arena; - - ExecutionPath path; - path.push_back(ExpressionStep::MakeConstant( - cel::ErrorValue::From(absl::InternalError("error"), &arena))); - path.push_back(ExpressionStep::MakeNotStrictlyFalseStep()); - - cel::runtime_internal::RuntimeTypeProvider type_provider( - cel::internal::GetTestingDescriptorPool()); - FlatExpressionEvaluatorState state( - 2, 0, type_provider, cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena); - cel::Activation activation; - cel::RuntimeOptions options; - ExecutionFrame frame(path, activation, options, state); - ASSERT_OK_AND_ASSIGN(cel::Value value, frame.Evaluate()); - ASSERT_TRUE(value.IsBool()); - EXPECT_TRUE(value.GetBool().NativeValue()); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/optional_or_step.cc b/eval/eval/optional_or_step.cc index 983f229b1..c0b341b4f 100644 --- a/eval/eval/optional_or_step.cc +++ b/eval/eval/optional_or_step.cc @@ -14,7 +14,6 @@ #include "eval/eval/optional_or_step.h" -#include #include #include #include @@ -24,11 +23,9 @@ #include "absl/types/span.h" #include "common/value.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" #include "eval/eval/expression_step_logic.h" -#include "internal/status_macros.h" #include "runtime/internal/errors.h" #include "google/protobuf/arena.h" @@ -130,96 +127,6 @@ void OptionalOrStep::Evaluate(ExecutionFrame* frame) const { frame->value_stack().PopAndPush(2, std::move(result), std::move(result_attr)); } -class ExhaustiveDirectOptionalOrStep : public DirectExpressionStep { - public: - ExhaustiveDirectOptionalOrStep( - int64_t expr_id, std::unique_ptr optional, - std::unique_ptr alternative, OptionalOrKind kind) - - : DirectExpressionStep(expr_id), - kind_(kind), - optional_(std::move(optional)), - alternative_(std::move(alternative)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; - - private: - OptionalOrKind kind_; - std::unique_ptr optional_; - std::unique_ptr alternative_; -}; - -absl::Status ExhaustiveDirectOptionalOrStep::Evaluate( - ExecutionFrameBase& frame, Value& result, AttributeTrail& attribute) const { - CEL_RETURN_IF_ERROR(optional_->Evaluate(frame, result, attribute)); - Value rhs; - AttributeTrail rhs_attr; - CEL_RETURN_IF_ERROR(alternative_->Evaluate(frame, rhs, rhs_attr)); - CEL_RETURN_IF_ERROR(EvalOptionalOr(kind_, result, rhs, attribute, rhs_attr, - result, attribute, frame.arena())); - return absl::OkStatus(); -} - -class DirectOptionalOrStep : public DirectExpressionStep { - public: - DirectOptionalOrStep(int64_t expr_id, - std::unique_ptr optional, - std::unique_ptr alternative, - OptionalOrKind kind) - - : DirectExpressionStep(expr_id), - kind_(kind), - optional_(std::move(optional)), - alternative_(std::move(alternative)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; - - private: - OptionalOrKind kind_; - std::unique_ptr optional_; - std::unique_ptr alternative_; -}; - -absl::Status DirectOptionalOrStep::Evaluate(ExecutionFrameBase& frame, - Value& result, - AttributeTrail& attribute) const { - CEL_RETURN_IF_ERROR(optional_->Evaluate(frame, result, attribute)); - - if (result.IsUnknown() || result.IsError()) { - // Forward the lhs error instead of attempting to evaluate the alternative - // (unlike CEL's commutative logic operators). - return absl::OkStatus(); - } - - auto optional_value = result.AsOptional(); - if (!optional_value.has_value()) { - result = MakeNoOverloadError(kind_, frame.arena()); - return absl::OkStatus(); - } - - if (optional_value->HasValue()) { - if (kind_ == OptionalOrKind::kOrValue) { - result = optional_value->Value(); - } - return absl::OkStatus(); - } - - CEL_RETURN_IF_ERROR(alternative_->Evaluate(frame, result, attribute)); - - // If optional.or check that rhs is an optional. - // - // Otherwise, we don't know what type to expect so can't check anything. - if (kind_ == OptionalOrKind::kOrOptional) { - if (!result.IsOptional() && !result.IsError() && !result.IsUnknown()) { - result = MakeNoOverloadError(kind_, frame.arena()); - } - } - - return absl::OkStatus(); -} - } // namespace void OptionalHasValueJumpStep::Evaluate(ExecutionFrame* frame) const { @@ -257,19 +164,4 @@ std::unique_ptr CreateOptionalOrStep(bool is_or_value) { is_or_value ? OptionalOrKind::kOrValue : OptionalOrKind::kOrOptional); } -std::unique_ptr CreateDirectOptionalOrStep( - int64_t expr_id, std::unique_ptr optional, - std::unique_ptr alternative, bool is_or_value, - bool short_circuiting) { - auto kind = - is_or_value ? OptionalOrKind::kOrValue : OptionalOrKind::kOrOptional; - if (short_circuiting) { - return std::make_unique(expr_id, std::move(optional), - std::move(alternative), kind); - } else { - return std::make_unique( - expr_id, std::move(optional), std::move(alternative), kind); - } -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/optional_or_step.h b/eval/eval/optional_or_step.h index f54057fa9..fc55a0ab7 100644 --- a/eval/eval/optional_or_step.h +++ b/eval/eval/optional_or_step.h @@ -15,14 +15,12 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_OPTIONAL_OR_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_OPTIONAL_OR_STEP_H_ -#include #include #include -#include "absl/status/status.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { @@ -54,13 +52,6 @@ std::unique_ptr CreateOptionalHasValueJumpStep( // optional.orValue. std::unique_ptr CreateOptionalOrStep(bool is_or_value); -// Creates a step implementing the short-circuiting optional.or or -// optional.orValue step. -std::unique_ptr CreateDirectOptionalOrStep( - int64_t expr_id, std::unique_ptr optional, - std::unique_ptr alternative, bool is_or_value, - bool short_circuiting); - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_OPTIONAL_OR_STEP_H_ diff --git a/eval/eval/optional_or_step_test.cc b/eval/eval/optional_or_step_test.cc deleted file mode 100644 index f0641ca6c..000000000 --- a/eval/eval/optional_or_step_test.cc +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "eval/eval/optional_or_step.h" - -#include - -#include "absl/memory/memory.h" -#include "absl/status/status.h" -#include "common/value.h" -#include "common/value_kind.h" -#include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" -#include "runtime/internal/errors.h" -#include "runtime/internal/runtime_type_provider.h" -#include "runtime/runtime_options.h" -#include "google/protobuf/arena.h" - -namespace google::api::expr::runtime { -namespace { - -using ::absl_testing::StatusIs; -using ::cel::Activation; -using ::cel::ErrorValue; -using ::cel::IntValue; -using ::cel::OptionalValue; -using ::cel::RuntimeOptions; -using ::cel::UnknownValue; -using ::cel::Value; -using ::cel::ValueKind; -using ::cel::test::ErrorValueIs; -using ::cel::test::IntValueIs; -using ::cel::test::OptionalValueIs; -using ::cel::test::ValueKindIs; -using ::testing::HasSubstr; -using ::testing::NiceMock; - -class MockDirectStep : public DirectExpressionStep { - public: - MOCK_METHOD(absl::Status, Evaluate, - (ExecutionFrameBase & frame, Value& result, - AttributeTrail& scratch), - (const, override)); -}; - -std::unique_ptr MockNeverCalledDirectStep() { - auto* mock = new NiceMock(); - EXPECT_CALL(*mock, Evaluate).Times(0); - return absl::WrapUnique(mock); -} - -std::unique_ptr MockExpectCallDirectStep() { - auto* mock = new NiceMock(); - EXPECT_CALL(*mock, Evaluate) - .Times(1) - .WillRepeatedly( - [](ExecutionFrameBase& frame, Value& result, AttributeTrail& attr) { - result = ErrorValue::From( - absl::InternalError("expected to be unused"), frame.arena()); - return absl::OkStatus(); - }); - return absl::WrapUnique(mock); -} - -class OptionalOrTest : public testing::Test { - public: - OptionalOrTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()) {} - - protected: - google::protobuf::Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; - Activation empty_activation_; -}; - -TEST_F(OptionalOrTest, OptionalOrLeftPresentShortcutRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(OptionalValue::Of(IntValue(42), &arena_)), - MockNeverCalledDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, OptionalValueIs(IntValueIs(42))); -} - -TEST_F(OptionalOrTest, OptionalOrLeftErrorShortcutsRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep( - ErrorValue::From(absl::InternalError("error"), &arena_)), - MockNeverCalledDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); -} - -TEST_F(OptionalOrTest, OptionalOrLeftErrorExhaustiveRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep( - ErrorValue::From(absl::InternalError("error"), &arena_)), - MockExpectCallDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/false); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); -} - -TEST_F(OptionalOrTest, OptionalOrLeftUnknownShortcutsRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(UnknownValue()), - MockNeverCalledDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST_F(OptionalOrTest, OptionalOrLeftUnknownExhaustiveRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(UnknownValue()), - MockExpectCallDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/false); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST_F(OptionalOrTest, OptionalOrLeftAbsentReturnRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(OptionalValue::None()), - CreateConstValueDirectStep(OptionalValue::Of(IntValue(42), &arena_)), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, OptionalValueIs(IntValueIs(42))); -} - -TEST_F(OptionalOrTest, OptionalOrLeftWrongType) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(IntValue(42)), - MockNeverCalledDirectStep(), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, - ErrorValueIs(StatusIs( - absl::StatusCode::kUnknown, - HasSubstr(cel::runtime_internal::kErrNoMatchingOverload)))); -} - -TEST_F(OptionalOrTest, OptionalOrRightWrongType) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(OptionalValue::None()), - CreateConstValueDirectStep(IntValue(42)), - /*is_or_value=*/false, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, - ErrorValueIs(StatusIs( - absl::StatusCode::kUnknown, - HasSubstr(cel::runtime_internal::kErrNoMatchingOverload)))); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftPresentShortcutRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(OptionalValue::Of(IntValue(42), &arena_)), - MockNeverCalledDirectStep(), - /*is_or_value=*/true, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, IntValueIs(42)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftPresentExhaustiveRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep(OptionalValue::Of(IntValue(42), &arena_)), - MockExpectCallDirectStep(), - /*is_or_value=*/true, - /*short_circuiting=*/false); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, IntValueIs(42)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftErrorShortcutsRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, - CreateConstValueDirectStep( - ErrorValue::From(absl::InternalError("error"), &arena_)), - MockNeverCalledDirectStep(), - /*is_or_value=*/true, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kError)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftUnknownShortcutsRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(UnknownValue()), - MockNeverCalledDirectStep(), true, true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - EXPECT_THAT(result, ValueKindIs(ValueKind::kUnknown)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftAbsentReturnRight) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(OptionalValue::None()), - CreateConstValueDirectStep(IntValue(42)), - /*is_or_value=*/true, - /*short_circuiting=*/true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, IntValueIs(42)); -} - -TEST_F(OptionalOrTest, OptionalOrValueLeftWrongType) { - RuntimeOptions options; - ExecutionFrameBase frame(empty_activation_, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectOptionalOrStep( - /*expr_id=*/-1, CreateConstValueDirectStep(IntValue(42)), - MockNeverCalledDirectStep(), true, true); - - Value result; - AttributeTrail scratch; - - ASSERT_OK(step->Evaluate(frame, result, scratch)); - - EXPECT_THAT(result, - ErrorValueIs(StatusIs( - absl::StatusCode::kUnknown, - HasSubstr(cel::runtime_internal::kErrNoMatchingOverload)))); -} - -} // namespace -} // namespace google::api::expr::runtime diff --git a/eval/eval/regex_match_step.cc b/eval/eval/regex_match_step.cc index d13e4f651..3c949ee9f 100644 --- a/eval/eval/regex_match_step.cc +++ b/eval/eval/regex_match_step.cc @@ -14,7 +14,6 @@ #include "eval/eval/regex_match_step.h" -#include #include #include #include @@ -24,11 +23,9 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" -#include "internal/status_macros.h" +#include "eval/eval/expression_step_logic.h" #include "re2/re2.h" namespace google::api::expr::runtime { @@ -37,7 +34,6 @@ namespace { using ::cel::BoolValue; using ::cel::StringValue; -using ::cel::Value; inline constexpr int kNumRegexMatchArguments = 1; inline constexpr size_t kRegexMatchStepSubject = 0; @@ -84,47 +80,8 @@ class RegexMatchStep final : public ExpressionStepBase { const std::shared_ptr re2_; }; -class RegexMatchDirectStep final : public DirectExpressionStep { - public: - RegexMatchDirectStep(int64_t expr_id, - std::unique_ptr subject, - std::shared_ptr re2) - : DirectExpressionStep(expr_id), - subject_(std::move(subject)), - re2_(std::move(re2)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - AttributeTrail subject_attr; - CEL_RETURN_IF_ERROR(subject_->Evaluate(frame, result, subject_attr)); - if (result.IsError() || result.IsUnknown()) { - return absl::OkStatus(); - } - - if (!result.IsString()) { - return absl::Status(absl::StatusCode::kInternal, - "First argument for regular " - "expression match must be a string"); - } - bool match = result.GetString().NativeValue(MatchesVisitor{*re2_}); - result = BoolValue(match); - return absl::OkStatus(); - } - - private: - std::unique_ptr subject_; - const std::shared_ptr re2_; -}; - } // namespace -std::unique_ptr CreateDirectRegexMatchStep( - int64_t expr_id, std::unique_ptr subject, - std::shared_ptr re2) { - return std::make_unique(expr_id, std::move(subject), - std::move(re2)); -} - absl::StatusOr> CreateRegexMatchStep( std::shared_ptr re2) { return std::make_unique(std::move(re2)); diff --git a/eval/eval/regex_match_step.h b/eval/eval/regex_match_step.h index bcbff7bdc..d44559787 100644 --- a/eval/eval/regex_match_step.h +++ b/eval/eval/regex_match_step.h @@ -15,20 +15,14 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_REGEX_MATCH_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_REGEX_MATCH_STEP_H_ -#include #include #include "absl/status/statusor.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" #include "re2/re2.h" namespace google::api::expr::runtime { -std::unique_ptr CreateDirectRegexMatchStep( - int64_t expr_id, std::unique_ptr subject, - std::shared_ptr re2); - absl::StatusOr> CreateRegexMatchStep( std::shared_ptr re2); } diff --git a/eval/eval/regex_match_step_test.cc b/eval/eval/regex_match_step_test.cc index 53b955b25..fa141365e 100644 --- a/eval/eval/regex_match_step_test.cc +++ b/eval/eval/regex_match_step_test.cc @@ -17,7 +17,6 @@ #include "cel/expr/checked.pb.h" #include "cel/expr/syntax.pb.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "eval/public/activation.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_expr_builder_factory.h" diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index a8f270698..9bfa5fad4 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -1,6 +1,5 @@ #include "eval/eval/select_step.h" -#include #include #include #include @@ -19,9 +18,9 @@ #include "common/value.h" #include "common/value_kind.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "eval/public/cel_value.h" #include "eval/public/structs/proto_message_type_adapter.h" #include "internal/status_macros.h" @@ -321,115 +320,6 @@ void SelectStep::Evaluate(ExecutionFrame* frame) const { frame->value_stack().PopAndPush(std::move(result), std::move(result_trail)); } -class DirectSelectStep : public DirectExpressionStep { - public: - DirectSelectStep(int64_t expr_id, - std::unique_ptr operand, - absl::string_view field, bool test_only, - bool enable_wrapper_type_null_unboxing, - bool enable_optional_types) - : DirectExpressionStep(expr_id), - operand_(std::move(operand)), - field_(field), - test_only_(test_only), - unboxing_option_(enable_wrapper_type_null_unboxing - ? ProtoWrapperTypeOptions::kUnsetNull - : ProtoWrapperTypeOptions::kUnsetProtoDefault), - enable_optional_types_(enable_optional_types) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override { - CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute)); - - if (result.IsError() || result.IsUnknown()) { - // Just forward. - return absl::OkStatus(); - } - - if (frame.attribute_tracking_enabled()) { - attribute = attribute.Step(&field_); - absl::optional value = CheckForMarkedAttributes(attribute, frame); - if (value.has_value()) { - result = std::move(value).value(); - return absl::OkStatus(); - } - } - - absl::optional optional_arg; - - if (enable_optional_types_ && result.IsOptional()) { - optional_arg = result.GetOptional(); - } - - switch (result.kind()) { - case ValueKind::kStruct: - case ValueKind::kMap: - break; - default: - if (optional_arg) { - break; - } - result = - cel::ErrorValue::From(InvalidSelectTargetError(), frame.arena()); - return absl::OkStatus(); - } - - if (test_only_) { - if (optional_arg) { - if (!optional_arg->HasValue()) { - result = cel::BoolValue{false}; - return absl::OkStatus(); - } - Value value; - optional_arg->Value(&value); - return PerformHas(value, field_, cel::StringValue::WrapUnsafe(field_), - frame.descriptor_pool(), frame.message_factory(), - frame.arena(), result); - } - return PerformHas(result, field_, cel::StringValue::WrapUnsafe(field_), - frame.descriptor_pool(), frame.message_factory(), - frame.arena(), result); - } - - if (optional_arg) { - if (!optional_arg->HasValue()) { - // result is still buffer for the container. just return. - return absl::OkStatus(); - } - Value value; - optional_arg->Value(&value); - auto status = PerformOptionalGet( - value, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, - frame.descriptor_pool(), frame.message_factory(), frame.arena(), - frame.options().enable_use_new_field_select_implementation, result); - if (!status.ok()) { - result = ErrorValue::From(std::move(status), frame.arena()); - } - return absl::OkStatus(); - } - - return PerformGet( - result, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, - frame.descriptor_pool(), frame.message_factory(), frame.arena(), - frame.options().enable_use_new_field_select_implementation, result); - } - - private: - std::unique_ptr operand_; - - // Field name in formats supported by each of the map and struct field access - // APIs. - // - // ToString or ValueManager::CreateString may force a copy so we do this at - // plan time. - std::string field_; - - // whether this is a has() expression. - bool test_only_; - ProtoWrapperTypeOptions unboxing_option_; - bool enable_optional_types_; -}; - bool CheckAttributeTrail(const std::string& field, ExecutionFrame* frame) { if (!frame->attribute_tracking_enabled()) { return false; @@ -619,15 +509,6 @@ void ProtoHasStep::EvaluateHas( } // namespace -std::unique_ptr CreateDirectSelectStep( - std::unique_ptr operand, absl::string_view field, - bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, - bool enable_optional_types) { - return std::make_unique( - expr_id, std::move(operand), std::move(field), test_only, - enable_wrapper_type_null_unboxing, enable_optional_types); -} - // Factory method for Select - based Execution step absl::StatusOr> CreateSelectStep( absl::string_view field, bool test_only, diff --git a/eval/eval/select_step.h b/eval/eval/select_step.h index 528e90627..9e5d2967b 100644 --- a/eval/eval/select_step.h +++ b/eval/eval/select_step.h @@ -1,23 +1,15 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_SELECT_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_SELECT_STEP_H_ -#include #include #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "common/type.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -// Factory method for recursively evaluated select step. -std::unique_ptr CreateDirectSelectStep( - std::unique_ptr operand, absl::string_view field, - bool test_only, int64_t expr_id, bool enable_wrapper_type_null_unboxing, - bool enable_optional_types = false); - // Factory method for Select stack machine based Execution step absl::StatusOr> CreateSelectStep( absl::string_view field, bool test_only, diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index 1fca8fa57..68c6f6f50 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -8,7 +8,6 @@ #include "cel/expr/syntax.pb.h" #include "google/protobuf/wrappers.pb.h" #include "absl/base/nullability.h" -#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/status/statusor.h" @@ -20,13 +19,8 @@ #include "common/expr.h" #include "common/legacy_value.h" #include "common/type.h" -#include "common/value.h" -#include "common/value_testing.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/ident_step.h" #include "eval/public/activation.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" @@ -37,16 +31,11 @@ #include "eval/public/unknown_set.h" #include "eval/testutil/test_extensions.pb.h" #include "eval/testutil/test_message.pb.h" -#include "extensions/protobuf/value.h" #include "internal/proto_matchers.h" #include "internal/status_macros.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "cel/expr/conformance/proto3/test_all_types.pb.h" #include "google/protobuf/descriptor.h" @@ -58,26 +47,16 @@ namespace { using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::Attribute; -using ::cel::AttributeQualifier; using ::cel::AttributeSet; -using ::cel::BoolValue; -using ::cel::ErrorValue; using ::cel::Expr; -using ::cel::IntValue; -using ::cel::OptionalValue; using ::cel::RuntimeOptions; using ::cel::TypeProvider; -using ::cel::UnknownValue; -using ::cel::Value; using ::cel::expr::conformance::proto3::TestAllTypes; -using ::cel::extensions::ProtoMessageToValue; using ::cel::internal::test::EqualsProto; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; -using ::cel::test::IntValueIs; using ::testing::Eq; using ::testing::HasSubstr; -using ::testing::UnorderedElementsAre; struct RunExpressionOptions { bool enable_unknowns = false; @@ -1089,537 +1068,6 @@ TEST_F(SelectStepTest, TypedSelectStepUnknownPatternResolvesToUnknown) { INSTANTIATE_TEST_SUITE_P(UnknownsEnabled, SelectStepConformanceTest, testing::Bool()); -class DirectSelectStepTest : public testing::Test { - public: - DirectSelectStepTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()) {} - - cel::Value TestWrapMessage(const google::protobuf::Message* message) { - CelValue value = CelProtoWrapper::CreateMessage(message, &arena_); - auto result = cel::interop_internal::FromLegacyValue(&arena_, value); - ABSL_DCHECK_OK(result.status()); - return std::move(result).value(); - } - - std::vector AttributeStrings(const UnknownValue& v) { - std::vector result; - for (const Attribute& attr : v.ToAttributeSet()) { - auto attr_str = attr.AsString(); - ABSL_DCHECK_OK(attr_str.status()); - result.push_back(std::move(attr_str).value()); - } - return result; - } - - protected: - google::protobuf::Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; -}; - -TEST_F(DirectSelectStepTest, SelectFromMap) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "one", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), - IsOk()); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), - IsOk()); - activation.InsertOrAssignValue("map_val", std::move(*map_builder).Build()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsInt()); - - EXPECT_EQ(result.GetInt().NativeValue(), 1); -} - -TEST_F(DirectSelectStepTest, HasMap) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "two", - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), - IsOk()); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), - IsOk()); - activation.InsertOrAssignValue("map_val", std::move(*map_builder).Build()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.GetBool()); - - EXPECT_TRUE(result.GetBool().NativeValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromOptionalMap) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "one", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), - IsOk()); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), - IsOk()); - activation.InsertOrAssignValue( - "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsOptional()); - EXPECT_THAT(result.GetOptional().Value(), IntValueIs(1)); -} - -TEST_F(DirectSelectStepTest, SelectFromOptionalMapAbsent) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "three", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), - IsOk()); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), - IsOk()); - activation.InsertOrAssignValue( - "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsOptional()); - EXPECT_FALSE(result.GetOptional().HasValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromOptionalStruct) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("struct_val", -1), - "single_int64", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - TestAllTypes message; - message.set_single_int64(1); - - ASSERT_OK_AND_ASSIGN( - Value struct_val, - ProtoMessageToValue(std::move(message), - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_)); - - activation.InsertOrAssignValue("struct_val", - OptionalValue::Of(struct_val, &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsOptional()); - EXPECT_THAT(result.GetOptional().Value(), IntValueIs(1)); -} - -TEST_F(DirectSelectStepTest, SelectFromOptionalStructFieldNotSet) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep(CreateDirectIdentStep("struct_val", -1), - "single_string", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - TestAllTypes message; - message.set_single_int64(1); - - ASSERT_OK_AND_ASSIGN( - Value struct_val, - ProtoMessageToValue(std::move(message), - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_)); - - activation.InsertOrAssignValue("struct_val", - OptionalValue::Of(struct_val, &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsOptional()); - EXPECT_FALSE(result.GetOptional().HasValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromEmptyOptional) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "one", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - activation.InsertOrAssignValue("map_val", OptionalValue::None()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsOptional()); - EXPECT_FALSE(result.GetOptional().HasValue()); -} - -TEST_F(DirectSelectStepTest, HasOptional) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "two", - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - auto map_builder = cel::NewMapValueBuilder(&arena_); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("one"), IntValue(1)), - IsOk()); - ASSERT_THAT( - map_builder->Put(cel::StringValue::WrapUnsafe("two"), IntValue(2)), - IsOk()); - activation.InsertOrAssignValue( - "map_val", OptionalValue::Of(std::move(*map_builder).Build(), &arena_)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsBool()); - - EXPECT_TRUE(result.GetBool().NativeValue()); -} - -TEST_F(DirectSelectStepTest, HasEmptyOptional) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("map_val", -1), "two", - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true, - /*enable_optional_types=*/true); - - activation.InsertOrAssignValue("map_val", OptionalValue::None()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsBool()); - - EXPECT_FALSE(result.GetBool().NativeValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromStruct) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("test_all_types", -1), "single_int64", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsInt()); - - EXPECT_EQ(result.GetInt().NativeValue(), 1); -} - -TEST_F(DirectSelectStepTest, HasStruct) { - cel::Activation activation; - RuntimeOptions options; - - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("test_all_types", -1), "single_string", - /*test_only=*/true, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - - // has(test_all_types.single_string) - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsBool()); - EXPECT_FALSE(result.GetBool().NativeValue()); -} - -TEST_F(DirectSelectStepTest, SelectFromUnsupportedType) { - cel::Activation activation; - RuntimeOptions options; - - auto step = - CreateDirectSelectStep(CreateDirectIdentStep("bool_val", -1), "one", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - activation.InsertOrAssignValue("bool_val", BoolValue(false)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsError()); - - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Applying SELECT to non-message type"))); -} - -TEST_F(DirectSelectStepTest, AttributeUpdatedIfRequested) { - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("test_all_types", -1), "single_int64", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsInt()); - EXPECT_EQ(result.GetInt().NativeValue(), 1); - - ASSERT_OK_AND_ASSIGN(std::string attr_str, attr.attribute().AsString()); - EXPECT_EQ(attr_str, "test_all_types.single_int64"); -} - -TEST_F(DirectSelectStepTest, MissingAttributesToErrors) { - cel::Activation activation; - RuntimeOptions options; - options.enable_missing_attribute_errors = true; - - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("test_all_types", -1), "single_int64", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - ASSERT_THAT(activation.SetMissingPatterns({cel::AttributePattern( - "test_all_types", - {cel::AttributeQualifierPattern::OfString("single_int64")})}), - IsOk()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("test_all_types.single_int64"))); -} - -TEST_F(DirectSelectStepTest, IdentifiesUnknowns) { - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - auto step = CreateDirectSelectStep( - CreateDirectIdentStep("test_all_types", -1), "single_int64", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - ASSERT_THAT(activation.SetUnknownPatterns({cel::AttributePattern( - "test_all_types", - {cel::AttributeQualifierPattern::OfString("single_int64")})}), - IsOk()); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsUnknown()); - - EXPECT_THAT(AttributeStrings(result.GetUnknown()), - UnorderedElementsAre("test_all_types.single_int64")); -} - -TEST_F(DirectSelectStepTest, ForwardErrorValue) { - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - auto step = CreateDirectSelectStep( - CreateConstValueDirectStep( - cel::ErrorValue::From(absl::InternalError("test1"), &arena_), -1), - "single_int64", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInternal, HasSubstr("test1"))); -} - -TEST_F(DirectSelectStepTest, ForwardUnknownOperand) { - cel::Activation activation; - RuntimeOptions options; - options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - - AttributeSet attr_set({Attribute("attr", {AttributeQualifier::OfInt(0)})}); - auto step = CreateDirectSelectStep( - CreateConstValueDirectStep(cel::common_internal::MakeUnknownValue( - cel::Unknown(std::move(attr_set))), - -1), - "single_int64", - /*test_only=*/false, -1, - /*enable_wrapper_type_null_unboxing=*/true); - - TestAllTypes message; - message.set_single_int64(1); - activation.InsertOrAssignValue("test_all_types", TestWrapMessage(&message)); - - ExecutionFrameBase frame(activation, options, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - Value result; - AttributeTrail attr; - ASSERT_THAT(step->Evaluate(frame, result, attr), IsOk()); - - ASSERT_TRUE(result.IsUnknown()); - EXPECT_THAT(AttributeStrings(result.GetUnknown()), - UnorderedElementsAre("attr[0]")); -} - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/shadowable_value_step.cc b/eval/eval/shadowable_value_step.cc index b6199ed7e..d257e7514 100644 --- a/eval/eval/shadowable_value_step.cc +++ b/eval/eval/shadowable_value_step.cc @@ -1,20 +1,16 @@ #include "eval/eval/shadowable_value_step.h" -#include #include #include #include -#include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" -#include "internal/status_macros.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { @@ -52,37 +48,6 @@ void ShadowableValueStep::Evaluate(ExecutionFrame* frame) const { } } -class DirectShadowableValueStep : public DirectExpressionStep { - public: - DirectShadowableValueStep(std::string identifier, cel::Value value, - int64_t expr_id) - : DirectExpressionStep(expr_id), - identifier_(std::move(identifier)), - value_(std::move(value)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; - - private: - std::string identifier_; - Value value_; -}; - -// TODO(uncreated-issue/67): Attribute tracking is skipped for the shadowed case. May -// cause problems for users with unknown tracking and variables named like -// 'list' etc, but follows the current behavior of the stack machine version. -absl::Status DirectShadowableValueStep::Evaluate( - ExecutionFrameBase& frame, Value& result, AttributeTrail& attribute) const { - CEL_ASSIGN_OR_RETURN(auto found, - frame.activation().FindVariable( - identifier_, frame.descriptor_pool(), - frame.message_factory(), frame.arena(), &result)); - if (!found) { - result = value_; - } - return absl::OkStatus(); -} - } // namespace absl::StatusOr> CreateShadowableValueStep( @@ -91,10 +56,4 @@ absl::StatusOr> CreateShadowableValueStep( std::move(value)); } -std::unique_ptr CreateDirectShadowableValueStep( - absl::string_view name, cel::Value value, int64_t expr_id) { - return std::make_unique(std::string(name), - std::move(value), expr_id); -} - } // namespace google::api::expr::runtime diff --git a/eval/eval/shadowable_value_step.h b/eval/eval/shadowable_value_step.h index fae90353b..c6f0a370a 100644 --- a/eval/eval/shadowable_value_step.h +++ b/eval/eval/shadowable_value_step.h @@ -1,14 +1,12 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_SHADOWABLE_VALUE_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_SHADOWABLE_VALUE_STEP_H_ -#include #include #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "common/value.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { @@ -18,9 +16,6 @@ namespace google::api::expr::runtime { absl::StatusOr> CreateShadowableValueStep( absl::string_view name, cel::Value value); -std::unique_ptr CreateDirectShadowableValueStep( - absl::string_view name, cel::Value value, int64_t expr_id); - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_SHADOWABLE_VALUE_STEP_H_ diff --git a/eval/eval/shadowable_value_step_test.cc b/eval/eval/shadowable_value_step_test.cc index 83ffd5857..42234b515 100644 --- a/eval/eval/shadowable_value_step_test.cc +++ b/eval/eval/shadowable_value_step_test.cc @@ -6,6 +6,7 @@ #include "absl/base/nullability.h" #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "base/type_provider.h" #include "common/value.h" #include "eval/eval/cel_expression_flat_impl.h" @@ -18,6 +19,7 @@ #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" #include "runtime/runtime_options.h" +#include "google/protobuf/arena.h" namespace google::api::expr::runtime { @@ -32,8 +34,8 @@ using ::testing::Eq; absl::StatusOr RunShadowableExpression( const absl_nonnull std::shared_ptr& env, - std::string identifier, cel::Value value, const Activation& activation, - Arena* arena) { + absl::string_view identifier, cel::Value value, + const Activation& activation, Arena* arena) { CEL_ASSIGN_OR_RETURN(auto step, CreateShadowableValueStep(identifier, std::move(value))); ExecutionPath path; diff --git a/eval/eval/ternary_step.cc b/eval/eval/ternary_step.cc index 0924b915f..d1ed0fd1e 100644 --- a/eval/eval/ternary_step.cc +++ b/eval/eval/ternary_step.cc @@ -1,20 +1,16 @@ #include "eval/eval/ternary_step.h" #include -#include #include #include #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "base/builtins.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" +#include "eval/eval/expression_step_logic.h" #include "eval/internal/errors.h" -#include "internal/status_macros.h" namespace google::api::expr::runtime { @@ -27,102 +23,6 @@ inline constexpr size_t kTernaryStepCondition = 0; inline constexpr size_t kTernaryStepTrue = 1; inline constexpr size_t kTernaryStepFalse = 2; -class ExhaustiveDirectTernaryStep : public DirectExpressionStep { - public: - ExhaustiveDirectTernaryStep(std::unique_ptr condition, - std::unique_ptr left, - std::unique_ptr right, - int64_t expr_id) - : DirectExpressionStep(expr_id), - condition_(std::move(condition)), - left_(std::move(left)), - right_(std::move(right)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute) const override { - cel::Value condition; - cel::Value lhs; - cel::Value rhs; - - AttributeTrail condition_attr; - AttributeTrail lhs_attr; - AttributeTrail rhs_attr; - - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - CEL_RETURN_IF_ERROR(left_->Evaluate(frame, lhs, lhs_attr)); - CEL_RETURN_IF_ERROR(right_->Evaluate(frame, rhs, rhs_attr)); - - if (condition.IsError() || condition.IsUnknown()) { - result = std::move(condition); - attribute = std::move(condition_attr); - return absl::OkStatus(); - } - - if (!condition.IsBool()) { - result = cel::ErrorValue::From(CreateNoMatchingOverloadError(kTernary), - frame.arena()); - return absl::OkStatus(); - } - - if (condition.GetBool().NativeValue()) { - result = std::move(lhs); - attribute = std::move(lhs_attr); - } else { - result = std::move(rhs); - attribute = std::move(rhs_attr); - } - return absl::OkStatus(); - } - - private: - std::unique_ptr condition_; - std::unique_ptr left_; - std::unique_ptr right_; -}; - -class ShortcircuitingDirectTernaryStep : public DirectExpressionStep { - public: - ShortcircuitingDirectTernaryStep( - std::unique_ptr condition, - std::unique_ptr left, - std::unique_ptr right, int64_t expr_id) - : DirectExpressionStep(expr_id), - condition_(std::move(condition)), - left_(std::move(left)), - right_(std::move(right)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& attribute) const override { - cel::Value condition; - - AttributeTrail condition_attr; - - CEL_RETURN_IF_ERROR(condition_->Evaluate(frame, condition, condition_attr)); - - if (condition.IsError() || condition.IsUnknown()) { - result = std::move(condition); - attribute = std::move(condition_attr); - return absl::OkStatus(); - } - - if (!condition.IsBool()) { - result = cel::ErrorValue::From(CreateNoMatchingOverloadError(kTernary), - frame.arena()); - return absl::OkStatus(); - } - - if (condition.GetBool().NativeValue()) { - return left_->Evaluate(frame, result, attribute); - } - return right_->Evaluate(frame, result, attribute); - } - - private: - std::unique_ptr condition_; - std::unique_ptr left_; - std::unique_ptr right_; -}; - class TernaryStep : public ExpressionStepBase { public: // Constructs FunctionStep that uses overloads specified. @@ -173,21 +73,6 @@ void TernaryStep::Evaluate(ExecutionFrame* frame) const { } // namespace -// Factory method for ternary (_?_:_) recursive execution step -std::unique_ptr CreateDirectTernaryStep( - std::unique_ptr condition, - std::unique_ptr left, - std::unique_ptr right, int64_t expr_id, - bool shortcircuiting) { - if (shortcircuiting) { - return std::make_unique( - std::move(condition), std::move(left), std::move(right), expr_id); - } - - return std::make_unique( - std::move(condition), std::move(left), std::move(right), expr_id); -} - std::unique_ptr CreateTernaryStep() { return std::make_unique(); } diff --git a/eval/eval/ternary_step.h b/eval/eval/ternary_step.h index 82491f589..75cf5d315 100644 --- a/eval/eval/ternary_step.h +++ b/eval/eval/ternary_step.h @@ -1,22 +1,12 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_TERNARY_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_TERNARY_STEP_H_ -#include #include -#include "absl/status/statusor.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" +#include "eval/eval/expression_step_logic.h" namespace google::api::expr::runtime { -// Factory method for ternary (_?_:_) recursive execution step -std::unique_ptr CreateDirectTernaryStep( - std::unique_ptr condition, - std::unique_ptr left, - std::unique_ptr right, int64_t expr_id, - bool shortcircuiting = true); - // Factory method for ternary (_?_:_) execution step std::unique_ptr CreateTernaryStep(); diff --git a/eval/eval/ternary_step_test.cc b/eval/eval/ternary_step_test.cc index 2d49700cc..5d5ac6ae4 100644 --- a/eval/eval/ternary_step_test.cc +++ b/eval/eval/ternary_step_test.cc @@ -7,27 +7,19 @@ #include "absl/base/nullability.h" #include "absl/status/status.h" -#include "base/attribute.h" #include "base/attribute_set.h" #include "base/type_provider.h" #include "common/expr.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" #include "eval/eval/cel_expression_flat_impl.h" -#include "eval/eval/const_value_step.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/public/activation.h" +#include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" #include "eval/public/unknown_attribute_set.h" #include "eval/public/unknown_set.h" #include "internal/testing.h" -#include "internal/testing_descriptor_pool.h" -#include "internal/testing_message_factory.h" -#include "runtime/activation.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" -#include "runtime/internal/runtime_type_provider.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -36,21 +28,13 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::absl_testing::StatusIs; -using ::cel::BoolValue; -using ::cel::ErrorValue; using ::cel::Expr; -using ::cel::IntValue; using ::cel::RuntimeOptions; using ::cel::TypeProvider; -using ::cel::UnknownValue; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::google::protobuf::Arena; -using ::testing::ElementsAre; using ::testing::Eq; -using ::testing::HasSubstr; -using ::testing::Truly; class LogicStepTest : public testing::TestWithParam { public: @@ -193,182 +177,6 @@ TEST_F(LogicStepTest, TestUnknownHandling) { INSTANTIATE_TEST_SUITE_P(LogicStepTest, LogicStepTest, testing::Bool()); -class TernaryStepDirectTest : public testing::TestWithParam { - public: - TernaryStepDirectTest() - : type_provider_(cel::internal::GetTestingDescriptorPool()) {} - - bool Shortcircuiting() { return GetParam(); } - - protected: - Arena arena_; - cel::runtime_internal::RuntimeTypeProvider type_provider_; -}; - -TEST_P(TernaryStepDirectTest, ReturnLhs) { - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(BoolValue(true), -1), - CreateConstValueDirectStep(IntValue(1), -1), - CreateConstValueDirectStep(IntValue(2), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - - ASSERT_TRUE(result.IsInt()); - EXPECT_EQ(result.GetInt().NativeValue(), 1); -} - -TEST_P(TernaryStepDirectTest, ReturnRhs) { - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(BoolValue(false), -1), - CreateConstValueDirectStep(IntValue(1), -1), - CreateConstValueDirectStep(IntValue(2), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - - ASSERT_TRUE(result.IsInt()); - EXPECT_EQ(result.GetInt().NativeValue(), 2); -} - -TEST_P(TernaryStepDirectTest, ForwardError) { - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - cel::Value error_value = - cel::ErrorValue::From(absl::InternalError("test error"), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(error_value, -1), - CreateConstValueDirectStep(IntValue(1), -1), - CreateConstValueDirectStep(IntValue(2), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kInternal, "test error")); -} - -TEST_P(TernaryStepDirectTest, ForwardUnknown) { - cel::Activation activation; - RuntimeOptions opts; - opts.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::vector attrs{{cel::Attribute("var")}}; - - cel::UnknownValue unknown_value = cel::common_internal::MakeUnknownValue( - cel::Unknown(cel::AttributeSet(attrs))); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(unknown_value, -1), - CreateConstValueDirectStep(IntValue(2), -1), - CreateConstValueDirectStep(IntValue(3), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - ASSERT_TRUE(result.IsUnknown()); - EXPECT_THAT(result.GetUnknown().ToAttributeSet(), - ElementsAre(Truly([](const cel::Attribute& attr) { - return attr.variable_name() == "var"; - }))); -} - -TEST_P(TernaryStepDirectTest, UnexpectedCondtionKind) { - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(IntValue(-1), -1), - CreateConstValueDirectStep(IntValue(1), -1), - CreateConstValueDirectStep(IntValue(2), -1), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(result.GetError().NativeValue(), - StatusIs(absl::StatusCode::kUnknown, - HasSubstr("No matching overloads found"))); -} - -TEST_P(TernaryStepDirectTest, Shortcircuiting) { - class RecordCallStep : public DirectExpressionStep { - public: - explicit RecordCallStep(bool& was_called) - : DirectExpressionStep(-1), was_called_(&was_called) {} - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& trail) const override { - *was_called_ = true; - result = IntValue(1); - return absl::OkStatus(); - } - - private: - bool* absl_nonnull was_called_; - }; - - bool lhs_was_called = false; - bool rhs_was_called = false; - - cel::Activation activation; - RuntimeOptions opts; - ExecutionFrameBase frame(activation, opts, type_provider_, - cel::internal::GetTestingDescriptorPool(), - cel::internal::GetTestingMessageFactory(), &arena_); - - std::unique_ptr step = CreateDirectTernaryStep( - CreateConstValueDirectStep(BoolValue(false), -1), - std::make_unique(lhs_was_called), - std::make_unique(rhs_was_called), -1, Shortcircuiting()); - - cel::Value result; - AttributeTrail attr_unused; - - ASSERT_THAT(step->Evaluate(frame, result, attr_unused), IsOk()); - - ASSERT_TRUE(result.IsInt()); - EXPECT_THAT(result.GetInt().NativeValue(), Eq(1)); - bool expect_eager_eval = !Shortcircuiting(); - EXPECT_EQ(lhs_was_called, expect_eager_eval); - EXPECT_TRUE(rhs_was_called); -} - -INSTANTIATE_TEST_SUITE_P(TernaryStepDirectTest, TernaryStepDirectTest, - testing::Bool()); - } // namespace } // namespace google::api::expr::runtime diff --git a/eval/eval/trace_step.h b/eval/eval/trace_step.h deleted file mode 100644 index cf4240248..000000000 --- a/eval/eval/trace_step.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_TRACE_STEP_H_ -#define THIRD_PARTY_CEL_CPP_EVAL_EVAL_TRACE_STEP_H_ - -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/types/optional.h" -#include "common/native_type.h" -#include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" -#include "eval/eval/evaluator_core.h" -#include "internal/status_macros.h" -namespace google::api::expr::runtime { - -// A decorator that implements tracing for recursively evaluated CEL -// expressions. -// -// Allows inspection for extensions to extract the wrapped expression. -class TraceStep : public DirectExpressionStep { - public: - explicit TraceStep(std::unique_ptr expression) - : DirectExpressionStep(-1), expression_(std::move(expression)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, cel::Value& result, - AttributeTrail& trail) const override { - CEL_RETURN_IF_ERROR(expression_->Evaluate(frame, result, trail)); - if (!frame.callback()) { - return absl::OkStatus(); - } - return frame.callback()(expression_->expr_id(), result, - frame.descriptor_pool(), frame.message_factory(), - frame.arena()); - } - - cel::NativeTypeId GetNativeTypeId() const override { - return cel::NativeTypeId::For(); - } - - absl::optional> GetDependencies() - const override { - return {{expression_.get()}}; - } - - absl::optional>> - ExtractDependencies() override { - std::vector> dependencies; - dependencies.push_back(std::move(expression_)); - return dependencies; - }; - - private: - std::unique_ptr expression_; -}; - -} // namespace google::api::expr::runtime - -#endif // THIRD_PARTY_CEL_CPP_EVAL_EVAL_TRACE_STEP_H_ diff --git a/eval/public/BUILD b/eval/public/BUILD index 39ab6368d..7bb6aac0d 100644 --- a/eval/public/BUILD +++ b/eval/public/BUILD @@ -58,7 +58,6 @@ cc_test( ":message_wrapper", "//eval/public/structs:trivial_legacy_type_info", "//eval/testutil:test_message_cc_proto", - "//internal:casts", "//internal:testing", "@com_google_protobuf//:protobuf", ], @@ -71,9 +70,7 @@ cc_library( ], deps = [ ":message_wrapper", - "//internal:casts", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/numeric:bits", "@com_google_absl//absl/types:variant", "@com_google_protobuf//:protobuf", ], diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index d9ebaab13..e7981a226 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -188,18 +188,14 @@ struct InterpreterOptions { // -1 means unbounded. // 0 means disabled (using a heap-based stack machine instead), which is the // default. - ABSL_DEPRECATED( - "The ability to enable recursive planning is being removed in the " - "near future") + ABSL_DEPRECATED("Recursive evaluation has been removed") int max_recursion_depth = 0; // Enable tracing support for recursively planned programs. // // Unlike the stack machine implementation, supporting tracing can affect // performance whether or not tracing is requested for a given evaluation. - ABSL_DEPRECATED( - "The ability to enable recursive planning is being removed in the " - "near future") + ABSL_DEPRECATED("Recursive evaluation has been removed") bool enable_recursive_tracing = false; // Enable fast implementations for some CEL standard functions. diff --git a/extensions/BUILD b/extensions/BUILD index 908cacb18..46fc088ae 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -171,7 +171,6 @@ cc_test( "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", @@ -337,7 +336,6 @@ cc_library( "//eval/compiler:flat_expr_builder", "//eval/compiler:flat_expr_builder_extensions", "//eval/eval:attribute_trail", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", "//eval/eval:expression_step_base", "//eval/public:cel_value", diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 07e0ae4c0..29cb5d6ff 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -51,7 +51,6 @@ #include "eval/compiler/flat_expr_builder.h" #include "eval/compiler/flat_expr_builder_extensions.h" #include "eval/eval/attribute_trail.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" #include "eval/public/cel_value.h" @@ -80,7 +79,6 @@ using ::cel::ExprKind; using ::cel::SelectExpr; using ::google::api::expr::runtime::AttributeTrail; using ::google::api::expr::runtime::CelValue; -using ::google::api::expr::runtime::DirectExpressionStep; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionFrameBase; using ::google::api::expr::runtime::ExpressionStep; @@ -823,58 +821,6 @@ void StackMachineImpl::Evaluate(ExecutionFrame* frame) const { frame->value_stack().Push(*std::move(result), std::move(attribute_trail)); } -class RecursiveImpl : public DirectExpressionStep { - public: - RecursiveImpl(int64_t expr_id, std::unique_ptr operand, - OptimizedSelectImpl impl) - : DirectExpressionStep(expr_id), - operand_(std::move(operand)), - impl_(std::move(impl)) {} - - absl::Status Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const override; - - private: - // Get the effective attribute for the optimized select expression. - // Assumes the operand is the top of stack if the attribute wasn't known at - // plan time. - AttributeTrail GetAttributeTrail(const AttributeTrail& operand_trail) const; - std::unique_ptr operand_; - OptimizedSelectImpl impl_; -}; - -AttributeTrail RecursiveImpl::GetAttributeTrail( - const AttributeTrail& operand_trail) const { - return impl_.GetAttributeTrail(operand_trail); -} - -absl::Status RecursiveImpl::Evaluate(ExecutionFrameBase& frame, Value& result, - AttributeTrail& attribute) const { - CEL_RETURN_IF_ERROR(operand_->Evaluate(frame, result, attribute)); - - if (result.IsError() || result.IsUnknown()) { - // Just forward. - return absl::OkStatus(); - } - - if (frame.attribute_tracking_enabled()) { - attribute = impl_.GetAttributeTrail(attribute); - CEL_ASSIGN_OR_RETURN(auto value, - CheckForMarkedAttributes(frame, attribute)); - if (value.has_value()) { - result = std::move(value).value(); - return absl::OkStatus(); - } - } - - if (!result.IsStruct()) { - return absl::InvalidArgumentError( - "Expected struct type for select optimization"); - } - CEL_ASSIGN_OR_RETURN(result, impl_.ApplySelect(frame, result.GetStruct())); - return absl::OkStatus(); -} - class SelectOptimizer : public ProgramOptimizer { public: explicit SelectOptimizer(const SelectOptimizationOptions& options) @@ -956,19 +902,6 @@ absl::Status SelectOptimizer::OnPostVisit(PlannerContext& context, OptimizedSelectImpl impl(std::move(instructions), std::move(qualifiers), presence_test, options_); - if (subexpression->IsRecursive()) { - auto program = subexpression->ExtractRecursiveProgram(); - auto deps = program.step->ExtractDependencies(); - if (!deps.has_value() || deps->empty()) { - return absl::InvalidArgumentError("Unexpected cel.@attribute call"); - } - subexpression->set_recursive_program( - std::make_unique(node.id(), std::move(deps->at(0)), - std::move(impl)), - program.depth); - return absl::OkStatus(); - } - google::api::expr::runtime::ExecutionPath path; // else, we need to preserve the original plan for the first argument. diff --git a/runtime/BUILD b/runtime/BUILD index 08e137669..d8ba3eb7d 100644 --- a/runtime/BUILD +++ b/runtime/BUILD @@ -316,7 +316,6 @@ cc_test( "//parser", "//parser:macro_registry", "//parser:standard_macros", - "//runtime/internal:runtime_impl", "@com_google_absl//absl/base:no_destructor", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", @@ -618,7 +617,6 @@ cc_test( "//internal:testing_descriptor_pool", "//parser", "//parser:options", - "//runtime/internal:runtime_impl", "@com_google_absl//absl/status", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/status:statusor", diff --git a/runtime/internal/BUILD b/runtime/internal/BUILD index 28703ef3b..cbe1fd2fe 100644 --- a/runtime/internal/BUILD +++ b/runtime/internal/BUILD @@ -62,11 +62,7 @@ cc_library( "//common:native_type", "//common:value", "//eval/compiler:flat_expr_builder", - "//eval/eval:attribute_trail", - "//eval/eval:comprehension_slots", - "//eval/eval:direct_expression_step", "//eval/eval:evaluator_core", - "//internal:casts", "//internal:status_macros", "//internal:well_known_types", "//runtime", @@ -220,7 +216,6 @@ cc_library( "//common:descriptor_pool_type_introspector", "//common:type", "//common:value", - "//internal:status_macros", "@com_google_absl//absl/base:nullability", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", diff --git a/runtime/internal/runtime_impl.cc b/runtime/internal/runtime_impl.cc index 3e8eef59a..8d8cda12b 100644 --- a/runtime/internal/runtime_impl.cc +++ b/runtime/internal/runtime_impl.cc @@ -21,13 +21,8 @@ #include "absl/status/statusor.h" #include "base/ast.h" #include "base/type_provider.h" -#include "common/native_type.h" #include "common/value.h" -#include "eval/eval/attribute_trail.h" -#include "eval/eval/comprehension_slots.h" -#include "eval/eval/direct_expression_step.h" #include "eval/eval/evaluator_core.h" -#include "internal/casts.h" #include "internal/status_macros.h" #include "runtime/activation_interface.h" #include "runtime/runtime.h" @@ -36,12 +31,7 @@ namespace cel::runtime_internal { namespace { -using ::google::api::expr::runtime::AttributeTrail; -using ::google::api::expr::runtime::ComprehensionSlots; -using ::google::api::expr::runtime::DirectExpressionStep; -using ::google::api::expr::runtime::ExecutionFrameBase; using ::google::api::expr::runtime::FlatExpression; -using ::google::api::expr::runtime::WrappedDirectStep; class ProgramImpl final : public TraceableProgram { public: @@ -76,46 +66,6 @@ class ProgramImpl final : public TraceableProgram { FlatExpression impl_; }; -class RecursiveProgramImpl final : public TraceableProgram { - public: - using EvaluationListener = TraceableProgram::EvaluationListener; - RecursiveProgramImpl( - const std::shared_ptr& environment, - FlatExpression impl, const DirectExpressionStep* absl_nonnull root) - : environment_(environment), impl_(std::move(impl)), root_(root) {} - - absl::StatusOr TraceImpl( - const ActivationInterface& activation, - EvaluationListener evaluation_listener, google::protobuf::Arena* absl_nonnull arena, - const EvaluateOptions& options) const override { - ABSL_DCHECK(arena != nullptr); - ComprehensionSlots slots(impl_.comprehension_slots_size()); - ExecutionFrameBase frame(activation, std::move(evaluation_listener), - impl_.options(), GetTypeProvider(), - environment_->descriptor_pool.get(), - options.message_factory != nullptr - ? options.message_factory - : environment_->MutableMessageFactory(), - arena, options.embedder_context, slots); - - Value result; - AttributeTrail attribute; - CEL_RETURN_IF_ERROR(root_->Evaluate(frame, result, attribute)); - - return result; - } - - const TypeProvider& GetTypeProvider() const override { - return environment_->type_registry.GetComposedTypeProvider(); - } - - private: - // Keep the Runtime environment alive while programs reference it. - std::shared_ptr environment_; - FlatExpression impl_; - const DirectExpressionStep* absl_nonnull root_; -}; - } // namespace absl::StatusOr> RuntimeImpl::CreateProgram( @@ -131,33 +81,7 @@ RuntimeImpl::CreateTraceableProgram( CEL_ASSIGN_OR_RETURN(auto flat_expr, expr_builder_.CreateExpressionImpl( std::move(ast), options.issues)); - // Special case if the program is fully recursive. - // - // This implementation avoids unnecessary allocs at evaluation time which - // improves performance notably for small expressions. - if (expr_builder_.options().max_recursion_depth != 0 && - !flat_expr.subexpressions().empty() && - // mainline expression is exactly one recursive step. - flat_expr.subexpressions().front().size() == 1 && - flat_expr.subexpressions().front().front().IsGenericStep() && - flat_expr.subexpressions() - .front() - .front() - .GetGenericStep() - ->GetNativeTypeId() == NativeTypeId::For()) { - const DirectExpressionStep* root = - internal::down_cast( - flat_expr.subexpressions().front().front().GetGenericStep()) - ->wrapped(); - return std::make_unique(environment_, - std::move(flat_expr), root); - } - return std::make_unique(environment_, std::move(flat_expr)); } -bool TestOnly_IsRecursiveImpl(const Program* program) { - return dynamic_cast(program) != nullptr; -} - } // namespace cel::runtime_internal diff --git a/runtime/internal/runtime_impl.h b/runtime/internal/runtime_impl.h index 7c5d445f9..ed5e5ad47 100644 --- a/runtime/internal/runtime_impl.h +++ b/runtime/internal/runtime_impl.h @@ -115,11 +115,6 @@ class RuntimeImpl : public Runtime { google::api::expr::runtime::FlatExprBuilder expr_builder_; }; -// Exposed for testing to validate program is recursively planned. -// -// Uses dynamic_casts to test. -bool TestOnly_IsRecursiveImpl(const Program* program); - } // namespace cel::runtime_internal #endif // THIRD_PARTY_CEL_CPP_RUNTIME_INTERNAL_RUNTIME_IMPL_H_ diff --git a/runtime/optional_types_test.cc b/runtime/optional_types_test.cc index 695f8900f..578b6d5bc 100644 --- a/runtime/optional_types_test.cc +++ b/runtime/optional_types_test.cc @@ -37,7 +37,6 @@ #include "parser/parser.h" #include "runtime/activation.h" #include "runtime/function.h" -#include "runtime/internal/runtime_impl.h" #include "runtime/reference_resolver.h" #include "runtime/runtime.h" #include "runtime/runtime_builder.h" @@ -185,41 +184,6 @@ class OptionalTypesTest bool EnableShortCircuiting() { return std::get<1>(GetParam()); } }; -TEST_P(OptionalTypesTest, RecursivePlan) { - RuntimeOptions opts; - opts.enable_qualified_type_identifiers = true; - opts.max_recursion_depth = -1; - opts.short_circuiting = EnableShortCircuiting(); - - const EvaluateResultTestCase& test_case = GetTestCase(); - - ASSERT_OK_AND_ASSIGN( - auto builder, - CreateStandardRuntimeBuilder(internal::GetTestingDescriptorPool(), opts)); - - ASSERT_OK(EnableOptionalTypes(builder)); - ASSERT_OK( - EnableReferenceResolver(builder, ReferenceResolverEnabled::kAlways)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, - Parse(test_case.expression, "", - ParserOptions{.enable_optional_syntax = true})); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr program, - ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - - EXPECT_TRUE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - - google::protobuf::Arena arena; - Activation activation; - - ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); - - EXPECT_THAT(result, test_case.value_matcher) << test_case.expression; -} - TEST_P(OptionalTypesTest, Defaults) { RuntimeOptions opts; opts.enable_qualified_type_identifiers = true; diff --git a/runtime/runtime_options.h b/runtime/runtime_options.h index 74ad8ecb7..af2f9d945 100644 --- a/runtime/runtime_options.h +++ b/runtime/runtime_options.h @@ -156,18 +156,14 @@ struct RuntimeOptions { // -1 means unbounded. // 0 means disabled (using a heap-based stack machine instead), which is the // default. - ABSL_DEPRECATED( - "The ability to enable recursive planning is being removed in the " - "near future") + ABSL_DEPRECATED("Recursive evaluation has been removed") int max_recursion_depth = 0; // Enable tracing support for recursively planned programs. // // Unlike the stack machine implementation, supporting tracing can affect // performance whether or not tracing is requested for a given evaluation. - ABSL_DEPRECATED( - "The ability to enable recursive planning is being removed in the " - "near future") + ABSL_DEPRECATED("Recursive evaluation has been removed") bool enable_recursive_tracing = false; // Enable fast implementations for some CEL standard functions. diff --git a/runtime/standard_runtime_builder_factory_test.cc b/runtime/standard_runtime_builder_factory_test.cc index 029897233..07bc7f1f9 100644 --- a/runtime/standard_runtime_builder_factory_test.cc +++ b/runtime/standard_runtime_builder_factory_test.cc @@ -38,7 +38,6 @@ #include "parser/parser.h" #include "parser/standard_macros.h" #include "runtime/activation.h" -#include "runtime/internal/runtime_impl.h" #include "runtime/runtime.h" #include "runtime/runtime_issue.h" #include "runtime/runtime_options.h" @@ -52,11 +51,9 @@ using ::absl_testing::IsOk; using ::absl_testing::StatusIs; using ::cel::extensions::ProtobufRuntimeAdapter; using ::cel::test::BoolValueIs; -using ::cel::test::IntValueIs; using ::cel::expr::ParsedExpr; using ::google::api::expr::parser::Parse; using ::testing::ElementsAre; -using ::testing::HasSubstr; using ::testing::TestWithParam; using ::testing::Truly; @@ -78,72 +75,6 @@ absl::StatusOr ParseWithTestMacros(absl::string_view expression) { return Parse(**src, GetMacros()); } -TEST(StandardRuntimeTest, RecursionLimitExceeded) { - RuntimeOptions opts; - opts.max_recursion_depth = 1; - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, ParseWithTestMacros("1 + 2")); - - EXPECT_THAT(ProtobufRuntimeAdapter::CreateProgram(*runtime, expr), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Maximum recursion depth of 1 exceeded"))); -} - -TEST(StandardRuntimeTest, RecursionUnderLimit) { - RuntimeOptions opts; - opts.max_recursion_depth = 2; - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, ParseWithTestMacros("1 + 2")); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr program, - ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - - // Whether the implementation is recursive shouldn't affect observable - // behavior, but it does have performance implications (it will skip - // allocating a value stack). - EXPECT_TRUE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - - google::protobuf::Arena arena; - Activation activation; - - ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); - EXPECT_THAT(result, IntValueIs(3)); -} - -TEST(StandardRuntimeTest, RecursionLimitTracksLazyExpressions) { - RuntimeOptions opts; - opts.max_recursion_depth = 8; - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, ParseWithTestMacros(R"cel( - cel.bind(a, 4 + (3 + (2 + 1)), - cel.bind(b, 7 + (6 + (5 + a)), - 9 + (8 + b) - ) - ))cel")); - - EXPECT_THAT(ProtobufRuntimeAdapter::CreateProgram(*runtime, expr), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("Maximum recursion depth of 8 exceeded"))); -} - struct EvaluateResultTestCase { std::string name; std::string expression; @@ -177,8 +108,6 @@ TEST_P(StandardRuntimeTest, Defaults) { ASSERT_OK_AND_ASSIGN(std::unique_ptr program, ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - EXPECT_FALSE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - google::protobuf::Arena arena; Activation activation; if (test_case.activation_builder != nullptr) { @@ -191,39 +120,6 @@ TEST_P(StandardRuntimeTest, Defaults) { << test_case.expression; } -TEST_P(StandardRuntimeTest, Recursive) { - RuntimeOptions opts; - opts.max_recursion_depth = -1; - const EvaluateResultTestCase& test_case = GetTestCase(); - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, - ParseWithTestMacros(test_case.expression)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr program, - ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - - // Whether the implementation is recursive shouldn't affect observable - // behavior, but it does have performance implications (it will skip - // allocating a value stack). - EXPECT_TRUE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - - google::protobuf::Arena arena; - Activation activation; - if (test_case.activation_builder != nullptr) { - ASSERT_THAT(test_case.activation_builder(activation), IsOk()); - } - - ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); - EXPECT_THAT(result, BoolValueIs(test_case.expected_result)) - << test_case.expression; -} - TEST_P(StandardRuntimeTest, FastBuiltins) { RuntimeOptions opts; opts.enable_fast_builtins = true; @@ -241,8 +137,6 @@ TEST_P(StandardRuntimeTest, FastBuiltins) { ASSERT_OK_AND_ASSIGN(std::unique_ptr program, ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - EXPECT_FALSE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - google::protobuf::Arena arena; Activation activation; if (test_case.activation_builder != nullptr) { @@ -255,40 +149,6 @@ TEST_P(StandardRuntimeTest, FastBuiltins) { << test_case.expression; } -TEST_P(StandardRuntimeTest, RecursiveFastBuiltins) { - RuntimeOptions opts; - opts.enable_fast_builtins = true; - opts.max_recursion_depth = -1; - const EvaluateResultTestCase& test_case = GetTestCase(); - - ASSERT_OK_AND_ASSIGN(auto builder, - CreateStandardRuntimeBuilder( - google::protobuf::DescriptorPool::generated_pool(), opts)); - - ASSERT_OK_AND_ASSIGN(auto runtime, std::move(builder).Build()); - - ASSERT_OK_AND_ASSIGN(ParsedExpr expr, - ParseWithTestMacros(test_case.expression)); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr program, - ProtobufRuntimeAdapter::CreateProgram(*runtime, expr)); - - // Whether the implementation is recursive shouldn't affect observable - // behavior, but it does have performance implications (it will skip - // allocating a value stack). - EXPECT_TRUE(runtime_internal::TestOnly_IsRecursiveImpl(program.get())); - - google::protobuf::Arena arena; - Activation activation; - if (test_case.activation_builder != nullptr) { - ASSERT_THAT(test_case.activation_builder(activation), IsOk()); - } - - ASSERT_OK_AND_ASSIGN(Value result, program->Evaluate(&arena, activation)); - EXPECT_THAT(result, BoolValueIs(test_case.expected_result)) - << test_case.expression; -} - INSTANTIATE_TEST_SUITE_P( Basic, StandardRuntimeTest, testing::ValuesIn(std::vector{ @@ -668,20 +528,11 @@ TEST(StandardRuntimeTest, RuntimeIssueSupport) { } } -enum class EvalStrategy { kIterative, kRecursive }; - -class StandardRuntimeEvalStrategyTest - : public ::testing::TestWithParam {}; +class StandardRuntimeEvalTest : public ::testing::Test {}; // Check that calls to specialized builtins are validated. -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinBoolOp) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinBoolOp) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } ASSERT_OK_AND_ASSIGN(auto builder, @@ -699,14 +550,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinBoolOp) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinTernaryOp) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinTernaryOp) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } ASSERT_OK_AND_ASSIGN(auto builder, @@ -743,14 +588,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinTernaryOp) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinIndex) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinIndex) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } ASSERT_OK_AND_ASSIGN(auto builder, @@ -771,15 +610,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinIndex) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinEq) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinEq) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } - ASSERT_OK_AND_ASSIGN(auto builder, CreateStandardRuntimeBuilder( @@ -799,14 +631,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinEq) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinIn) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, InvalidBuiltinIn) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } ASSERT_OK_AND_ASSIGN(auto builder, @@ -827,14 +653,8 @@ TEST_P(StandardRuntimeEvalStrategyTest, InvalidBuiltinIn) { StatusIs(absl::StatusCode::kInvalidArgument)); } -TEST_P(StandardRuntimeEvalStrategyTest, PrecisionPreservingDoubleFormat) { - EvalStrategy eval_strategy = GetParam(); +TEST_F(StandardRuntimeEvalTest, PrecisionPreservingDoubleFormat) { RuntimeOptions options; - if (eval_strategy == EvalStrategy::kRecursive) { - options.max_recursion_depth = -1; - } else { - options.max_recursion_depth = 0; - } options.enable_precision_preserving_double_format = true; @@ -861,12 +681,5 @@ TEST_P(StandardRuntimeEvalStrategyTest, PrecisionPreservingDoubleFormat) { } } -INSTANTIATE_TEST_SUITE_P( - StandardRuntimeEvalStrategyTest, StandardRuntimeEvalStrategyTest, - testing::Values(EvalStrategy::kIterative, EvalStrategy::kRecursive), - [](const auto& info) -> std::string { - return info.param == EvalStrategy::kIterative ? "Iterative" : "Recursive"; - }); - } // namespace } // namespace cel From 505315e6d90496371c3f860a29dfc6d5036c053d Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 9 Oct 2026 06:02:57 -0700 Subject: [PATCH 278/350] No public description PiperOrigin-RevId: 996487517 --- common/optional_ref.h | 1 + 1 file changed, 1 insertion(+) diff --git a/common/optional_ref.h b/common/optional_ref.h index c7ba580fc..f7f21b7a1 100644 --- a/common/optional_ref.h +++ b/common/optional_ref.h @@ -79,6 +79,7 @@ class optional_ref final { constexpr bool has_value() const { return value_ != nullptr; } + ABSL_DEPRECATE_AND_INLINE() constexpr explicit operator bool() const { return has_value(); } constexpr T& value() const { From 0e0ac39d63aaf1e757bf6ccea71e161767c29039 Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 9 Oct 2026 10:15:49 -0700 Subject: [PATCH 279/350] Add unknown benchmarks for the modern API PiperOrigin-RevId: 996606898 --- eval/tests/BUILD | 1 + eval/tests/modern_benchmark_test.cc | 175 ++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+) diff --git a/eval/tests/BUILD b/eval/tests/BUILD index b706101df..47d693e17 100644 --- a/eval/tests/BUILD +++ b/eval/tests/BUILD @@ -68,6 +68,7 @@ cc_test( ], deps = [ ":request_context_cc_proto", + "//base:attributes", "//checker:validation_result", "//common:decl", "//common:legacy_value", diff --git a/eval/tests/modern_benchmark_test.cc b/eval/tests/modern_benchmark_test.cc index de984617e..cdbb2f3f1 100644 --- a/eval/tests/modern_benchmark_test.cc +++ b/eval/tests/modern_benchmark_test.cc @@ -36,6 +36,7 @@ #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/string_view.h" +#include "base/attribute.h" #include "checker/validation_result.h" #include "common/decl.h" #include "common/native_type.h" @@ -80,6 +81,7 @@ using ::google::api::expr::parser::EnrichedParse; using ::google::api::expr::parser::Parse; using ::google::api::expr::runtime::RequestContext; using ::google::rpc::context::AttributeContext; +using ::testing::UnorderedElementsAre; RuntimeOptions GetOptions() { RuntimeOptions options; @@ -1465,6 +1467,179 @@ void BM_MapTransformComprehension(benchmark::State& state) { BENCHMARK(BM_MapTransformComprehension)->Range(1, 1 << 16); +void BM_SingleUnknownVariable(benchmark::State& state) { + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, Parse("unk")); + + RuntimeOptions options = GetOptions(); + options.unknown_processing = UnknownProcessingOptions::kAttributeOnly; + auto runtime = StandardRuntimeOrDie(options); + + ASSERT_OK_AND_ASSIGN(auto program, ProtobufRuntimeAdapter::CreateProgram( + *runtime, parsed_expr)); + + Activation activation; + ASSERT_THAT(activation.SetUnknownPatterns({AttributePattern("unk")}), IsOk()); + + for (auto _ : state) { + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + ASSERT_TRUE(result.IsUnknown()); + ASSERT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(Attribute("unk"))); + } +} + +BENCHMARK(BM_SingleUnknownVariable); + +void BM_DeepUnknownQualifier(benchmark::State& state) { + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + Parse("a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z")); + + RuntimeOptions options = GetOptions(); + options.unknown_processing = UnknownProcessingOptions::kAttributeOnly; + auto runtime = StandardRuntimeOrDie(options); + + ASSERT_OK_AND_ASSIGN(auto program, ProtobufRuntimeAdapter::CreateProgram( + *runtime, parsed_expr)); + + google::protobuf::Arena arena; + google::protobuf::Arena::UniquePtr struct_value = + google::protobuf::Arena::MakeUnique(&arena); + google::protobuf::Struct* struct_value_ptr = struct_value.get(); + for (char l = 'b'; l <= 'z'; ++l) { + auto* fields = struct_value_ptr->mutable_fields(); + struct_value_ptr = (*fields)[std::string(1, l)].mutable_struct_value(); + } + + Attribute attribute( + "a", + {AttributeQualifier::OfString("b"), AttributeQualifier::OfString("c"), + AttributeQualifier::OfString("d"), AttributeQualifier::OfString("e"), + AttributeQualifier::OfString("f"), AttributeQualifier::OfString("g"), + AttributeQualifier::OfString("h"), AttributeQualifier::OfString("i"), + AttributeQualifier::OfString("j"), AttributeQualifier::OfString("k"), + AttributeQualifier::OfString("l"), AttributeQualifier::OfString("m"), + AttributeQualifier::OfString("n"), AttributeQualifier::OfString("o"), + AttributeQualifier::OfString("p"), AttributeQualifier::OfString("q"), + AttributeQualifier::OfString("r"), AttributeQualifier::OfString("s"), + AttributeQualifier::OfString("t"), AttributeQualifier::OfString("u"), + AttributeQualifier::OfString("v"), AttributeQualifier::OfString("w"), + AttributeQualifier::OfString("x"), AttributeQualifier::OfString("y"), + AttributeQualifier::OfString("z")}); + + Activation activation; + ASSERT_THAT(activation.SetUnknownPatterns({AttributePattern( + "a", {AttributeQualifierPattern::OfString("b"), + AttributeQualifierPattern::OfString("c"), + AttributeQualifierPattern::OfString("d"), + AttributeQualifierPattern::OfString("e"), + AttributeQualifierPattern::OfString("f"), + AttributeQualifierPattern::OfString("g"), + AttributeQualifierPattern::OfString("h"), + AttributeQualifierPattern::OfString("i"), + AttributeQualifierPattern::OfString("j"), + AttributeQualifierPattern::OfString("k"), + AttributeQualifierPattern::OfString("l"), + AttributeQualifierPattern::OfString("m"), + AttributeQualifierPattern::OfString("n"), + AttributeQualifierPattern::OfString("o"), + AttributeQualifierPattern::OfString("p"), + AttributeQualifierPattern::OfString("q"), + AttributeQualifierPattern::OfString("r"), + AttributeQualifierPattern::OfString("s"), + AttributeQualifierPattern::OfString("t"), + AttributeQualifierPattern::OfString("u"), + AttributeQualifierPattern::OfString("v"), + AttributeQualifierPattern::OfString("w"), + AttributeQualifierPattern::OfString("x"), + AttributeQualifierPattern::OfString("y"), + AttributeQualifierPattern::OfString("z")})}), + IsOk()); + activation.InsertOrAssignValue( + "a", + cel::Value::WrapMessage(struct_value.get(), runtime->GetDescriptorPool(), + runtime->GetMessageFactory(), &arena)); + + for (auto _ : state) { + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + ASSERT_TRUE(result.IsUnknown()); + ASSERT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(attribute)); + } +} + +BENCHMARK(BM_DeepUnknownQualifier); + +void BM_ObnoxiousUnknownQualifier(benchmark::State& state) { + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + Parse("a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z")); + + RuntimeOptions options = GetOptions(); + options.unknown_processing = UnknownProcessingOptions::kAttributeOnly; + auto runtime = StandardRuntimeOrDie(options); + + ASSERT_OK_AND_ASSIGN(auto program, ProtobufRuntimeAdapter::CreateProgram( + *runtime, parsed_expr)); + + google::protobuf::Arena arena; + google::protobuf::Arena::UniquePtr struct_value = + google::protobuf::Arena::MakeUnique(&arena); + google::protobuf::Struct* struct_value_ptr = struct_value.get(); + std::vector attributes; + for (char v = 'a'; v <= 'z'; ++v) { + std::vector qualifiers; + for (char q = 'a'; q <= 'z'; ++q) { + if (v == q) { + continue; + } + qualifiers.push_back( + AttributeQualifierPattern::OfString(std::string(1, q))); + } + attributes.emplace_back(std::string(1, v), std::move(qualifiers)); + } + for (char l = 'b'; l <= 'z'; ++l) { + auto* fields = struct_value_ptr->mutable_fields(); + struct_value_ptr = (*fields)[std::string(1, l)].mutable_struct_value(); + } + Activation activation; + ASSERT_THAT(activation.SetUnknownPatterns(attributes), IsOk()); + activation.InsertOrAssignValue( + "a", + cel::Value::WrapMessage(struct_value.get(), runtime->GetDescriptorPool(), + runtime->GetMessageFactory(), &arena)); + Attribute attribute( + "a", + {AttributeQualifier::OfString("b"), AttributeQualifier::OfString("c"), + AttributeQualifier::OfString("d"), AttributeQualifier::OfString("e"), + AttributeQualifier::OfString("f"), AttributeQualifier::OfString("g"), + AttributeQualifier::OfString("h"), AttributeQualifier::OfString("i"), + AttributeQualifier::OfString("j"), AttributeQualifier::OfString("k"), + AttributeQualifier::OfString("l"), AttributeQualifier::OfString("m"), + AttributeQualifier::OfString("n"), AttributeQualifier::OfString("o"), + AttributeQualifier::OfString("p"), AttributeQualifier::OfString("q"), + AttributeQualifier::OfString("r"), AttributeQualifier::OfString("s"), + AttributeQualifier::OfString("t"), AttributeQualifier::OfString("u"), + AttributeQualifier::OfString("v"), AttributeQualifier::OfString("w"), + AttributeQualifier::OfString("x"), AttributeQualifier::OfString("y"), + AttributeQualifier::OfString("z")}); + + for (auto _ : state) { + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(cel::Value result, + program->Evaluate(&arena, activation)); + ASSERT_TRUE(result.IsUnknown()); + ASSERT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(attribute)); + } +} + +BENCHMARK(BM_ObnoxiousUnknownQualifier); + } // namespace } // namespace cel From ac085413ea08828982ed6ee30de90bf15b7bfefd Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 9 Oct 2026 11:39:31 -0700 Subject: [PATCH 280/350] Implement machinery for new unknown attribute and unknown function implementation PiperOrigin-RevId: 996659108 --- base/BUILD | 1 + base/attribute.cc | 117 ++++++-- base/attribute.h | 73 ++++- base/attribute_test.cc | 71 ++++- common/internal/BUILD | 32 ++ common/internal/unknowns.cc | 265 +++++++++++++++++ common/internal/unknowns.h | 477 ++++++++++++++++++++++++++++++ common/internal/unknowns_test.cc | 274 +++++++++++++++++ eval/public/cel_attribute_test.cc | 3 +- 9 files changed, 1277 insertions(+), 36 deletions(-) create mode 100644 common/internal/unknowns.cc create mode 100644 common/internal/unknowns.h create mode 100644 common/internal/unknowns_test.cc diff --git a/base/BUILD b/base/BUILD index f4b5fcfd5..cf116727d 100644 --- a/base/BUILD +++ b/base/BUILD @@ -33,6 +33,7 @@ cc_library( ], deps = [ ":kind", + "//internal:lexis", "//internal:status_macros", "//internal:strings", "@com_google_absl//absl/algorithm:container", diff --git a/base/attribute.cc b/base/attribute.cc index 768e7c4f4..0375067d8 100644 --- a/base/attribute.cc +++ b/base/attribute.cc @@ -30,6 +30,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "base/kind.h" +#include "internal/lexis.h" #include "internal/status_macros.h" #include "internal/strings.h" @@ -133,14 +134,61 @@ struct AttributeQualifierToString { } void operator()(absl::string_view value) const { - absl::StrAppend(&output, internal::FormatStringLiteral(value)); + if (internal::LexisIsIdentifier(value)) { + absl::StrAppend(&output, value); + } else { + internal::FormatStringLiteralTo(value, &output); + } + } + + void operator()(common_internal::WildcardType) const { + output.push_back('*'); + } + + std::string& output; +}; + +struct AttributeQualifierJoinString { + void operator()(std::monostate) const {} + + void operator()(bool value) const { + output.push_back('['); + output.append(value ? "true" : "false"); + output.push_back(']'); + } + + void operator()(int64_t value) const { + absl::StrAppend(&output, "[", value, "]"); } - void operator()(common_internal::WildcardType) const { output.append("*"); } + void operator()(uint64_t value) const { + absl::StrAppend(&output, "[", value, "u", "]"); + } + + void operator()(absl::string_view value) const { + if (internal::LexisIsIdentifier(value)) { + absl::StrAppend(&output, ".", value); + } else { + output.push_back('['); + internal::FormatStringLiteralTo(value, &output); + output.push_back(']'); + } + } + + void operator()(common_internal::WildcardType) const { output.append(".*"); } std::string& output; }; +template +void AttributeQualifierAppendJoinString(std::string* out, + absl::Span qualifiers) { + for (const auto& qualifier : qualifiers) { + std::visit(AttributeQualifierJoinString{*out}, + common_internal::AsVariant(qualifier)); + } +} + } // namespace Kind AttributeQualifier::kind() const { @@ -218,34 +266,61 @@ absl::StatusOr Attribute::AsString() const { std::string Attribute::ToString() const { std::string result; result.append(variable_name()); - for (const auto& qualifier : qualifier_path()) { - result.push_back('['); - std::visit(AttributeQualifierToString{result}, - common_internal::AsVariant(qualifier)); - result.push_back(']'); - } + AttributeQualifier::AppendJoinString(&result, qualifier_path()); return result; } std::string AttributePattern::ToString() const { std::string result; result.append(variable()); - for (const auto& qualifier : qualifier_path()) { - result.push_back('['); - std::visit(AttributeQualifierToString{result}, - common_internal::AsVariant(qualifier)); - result.push_back(']'); - } + AttributeQualifierPattern::AppendJoinString(&result, qualifier_path()); return result; } -std::string AttributeQualifier::ToString() const { - std::string result; - absl::Status status = - std::visit(AttributeQualifierStringPrinter{&result}, value_); - ABSL_DCHECK_OK(status) << "bad attribute qualifier"; - status.IgnoreError(); - return result; +void AttributeQualifier::AppendToString(std::string* absl_nonnull out) const { + std::visit(AttributeQualifierToString{*out}, value_); +} + +void AttributeQualifierView::AppendToString( + std::string* absl_nonnull out) const { + std::visit(AttributeQualifierToString{*out}, value_); +} + +void AttributeQualifierPattern::AppendToString( + std::string* absl_nonnull out) const { + std::visit(AttributeQualifierToString{*out}, value_); +} + +void AttributeQualifier::AppendJoinString(std::string* absl_nonnull out) const { + std::visit(AttributeQualifierJoinString{*out}, value_); +} + +void AttributeQualifierView::AppendJoinString( + std::string* absl_nonnull out) const { + std::visit(AttributeQualifierJoinString{*out}, value_); +} + +void AttributeQualifierPattern::AppendJoinString( + std::string* absl_nonnull out) const { + std::visit(AttributeQualifierJoinString{*out}, value_); +} + +void AttributeQualifier::AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers) { + AttributeQualifierAppendJoinString(out, qualifiers); +} + +void AttributeQualifierView::AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers) { + AttributeQualifierAppendJoinString(out, qualifiers); +} + +void AttributeQualifierPattern::AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers) { + AttributeQualifierAppendJoinString(out, qualifiers); } bool AttributeQualifier::IsMatch(const AttributeQualifier& other) const { diff --git a/base/attribute.h b/base/attribute.h index c4439a176..2229dc495 100644 --- a/base/attribute.h +++ b/base/attribute.h @@ -27,6 +27,7 @@ #include "absl/base/attributes.h" #include "absl/base/macros.h" +#include "absl/base/nullability.h" #include "absl/functional/overload.h" #include "absl/log/absl_check.h" #include "absl/status/statusor.h" @@ -38,8 +39,10 @@ namespace cel { namespace common_internal { -class AttributeMatcherNode; struct WildcardType {}; +// Important. The types in the variants must align with each other and the order +// matters. It must be int < uint < string < bool. If you change it here in the +// header you must change the logic in the source. using AttributeQualifierVariant = std::variant; using AttributeQualifierPatternVariant = @@ -115,9 +118,6 @@ class AttributeQualifier { Kind kind() const; - [[nodiscard]] - std::string ToString() const; - ABSL_DEPRECATE_AND_INLINE() std::optional GetInt64Key() const { return AsInt(); } @@ -216,6 +216,27 @@ class AttributeQualifier { return std::nullopt; } + void AppendToString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string ToString() const { + std::string out; + AppendToString(&out); + return out; + } + + void AppendJoinString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string JoinString() const { + std::string out; + AppendJoinString(&out); + return out; + } + + static void AppendJoinString(std::string* absl_nonnull out, + absl::Span qualifiers); + [[nodiscard]] bool IsMatch(const AttributeQualifier& other) const; @@ -392,6 +413,28 @@ class AttributeQualifierView { return std::nullopt; } + void AppendToString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string ToString() const { + std::string out; + AppendToString(&out); + return out; + } + + void AppendJoinString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string JoinString() const { + std::string out; + AppendJoinString(&out); + return out; + } + + static void AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers); + private: friend const common_internal::AttributeQualifierViewVariant& common_internal::AsVariant(const AttributeQualifierView& qualifier); @@ -633,6 +676,28 @@ class AttributeQualifierPattern { value_); } + void AppendToString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string ToString() const { + std::string out; + AppendToString(&out); + return out; + } + + void AppendJoinString(std::string* absl_nonnull out) const; + + [[nodiscard]] + std::string JoinString() const { + std::string out; + AppendJoinString(&out); + return out; + } + + static void AppendJoinString( + std::string* absl_nonnull out, + absl::Span qualifiers); + [[nodiscard]] bool IsMatch(const AttributeQualifier& qualifier) const; diff --git a/base/attribute_test.cc b/base/attribute_test.cc index 0adff75ad..945a8569a 100644 --- a/base/attribute_test.cc +++ b/base/attribute_test.cc @@ -14,6 +14,7 @@ #include "base/attribute.h" +#include #include #include "absl/strings/string_view.h" @@ -80,19 +81,71 @@ TEST(Attribute, ToString) { EXPECT_EQ(Attribute("foo", {AttributeQualifier::OfBool(true), AttributeQualifier::OfInt(2), AttributeQualifier::OfUint(3), - AttributeQualifier::OfString("bar")}) + AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz qux")}) .ToString(), - "foo[true][2][3u][\"bar\"]"); + "foo[true][2][3u].bar[\"baz qux\"]"); } TEST(AttributePattern, ToString) { - EXPECT_EQ(AttributePattern("foo", {AttributeQualifierPattern::OfBool(true), - AttributeQualifierPattern::OfInt(2), - AttributeQualifierPattern::OfUint(3), - AttributeQualifierPattern::OfString("bar"), - AttributeQualifierPattern::Wildcard()}) - .ToString(), - "foo[true][2][3u][\"bar\"][*]"); + EXPECT_EQ( + AttributePattern("foo", {AttributeQualifierPattern::OfBool(true), + AttributeQualifierPattern::OfInt(2), + AttributeQualifierPattern::OfUint(3), + AttributeQualifierPattern::OfString("bar"), + AttributeQualifierPattern::OfString("baz qux"), + AttributeQualifierPattern::Wildcard()}) + .ToString(), + "foo[true][2][3u].bar[\"baz qux\"].*"); +} + +TEST(AttributeQualifierView, Order) { + // int < uint < string < bool + EXPECT_LT(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifierView::OfUint(uint64_t{0})); + EXPECT_LT(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifierView::OfString("")); + EXPECT_LT(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifierView::OfBool(false)); + EXPECT_LT(AttributeQualifierView::OfUint(uint64_t{0}), + AttributeQualifierView::OfString("")); + EXPECT_LT(AttributeQualifierView::OfUint(uint64_t{0}), + AttributeQualifierView::OfBool(false)); +} + +TEST(AttributeQualifierView, TransparentOrder) { + EXPECT_EQ(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifier::OfInt(0)); + EXPECT_EQ(AttributeQualifierView::OfUint(uint64_t{0}), + AttributeQualifier::OfUint(0)); + EXPECT_EQ(AttributeQualifierView::OfString("0"), + AttributeQualifier::OfString("0")); + EXPECT_EQ(AttributeQualifierView::OfBool(false), + AttributeQualifier::OfBool(false)); + EXPECT_EQ(AttributeQualifier::OfInt(0), + AttributeQualifierView::OfInt(int64_t{0})); + EXPECT_EQ(AttributeQualifier::OfUint(0), + AttributeQualifierView::OfUint(uint64_t{0})); + EXPECT_EQ(AttributeQualifier::OfString("0"), + AttributeQualifierView::OfString("0")); + EXPECT_EQ(AttributeQualifier::OfBool(false), + AttributeQualifierView::OfBool(false)); + EXPECT_LT(AttributeQualifierView::OfInt(int64_t{0}), + AttributeQualifier::OfInt(1)); + EXPECT_LT(AttributeQualifierView::OfUint(uint64_t{0}), + AttributeQualifier::OfUint(1)); + EXPECT_LT(AttributeQualifierView::OfString("0"), + AttributeQualifier::OfString("1")); + EXPECT_LT(AttributeQualifierView::OfBool(false), + AttributeQualifier::OfBool(true)); + EXPECT_LT(AttributeQualifier::OfInt(0), + AttributeQualifierView::OfInt(int64_t{1})); + EXPECT_LT(AttributeQualifier::OfUint(0), + AttributeQualifierView::OfUint(uint64_t{1})); + EXPECT_LT(AttributeQualifier::OfString("0"), + AttributeQualifierView::OfString("1")); + EXPECT_LT(AttributeQualifier::OfBool(false), + AttributeQualifierView::OfBool(true)); } } // namespace diff --git a/common/internal/BUILD b/common/internal/BUILD index 99097fb12..c3b92545d 100644 --- a/common/internal/BUILD +++ b/common/internal/BUILD @@ -17,6 +17,38 @@ load("@rules_cc//cc:cc_test.bzl", "cc_test") package(default_visibility = ["//visibility:public"]) +cc_library( + name = "unknowns", + srcs = ["unknowns.cc"], + hdrs = ["unknowns.h"], + deps = [ + "//base:attribute_matcher", + "//base:attributes", + "//internal:arena_tree", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/functional:overload", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings:string_view", + "@com_google_protobuf//:protobuf", + ], +) + +cc_test( + name = "unknowns_test", + srcs = ["unknowns_test.cc"], + deps = [ + ":unknowns", + "//base:attribute_matcher", + "//base:attributes", + "//internal:testing", + "@com_google_absl//absl/status:status_matchers", + "@com_google_protobuf//:protobuf", + ], +) + cc_library( name = "byte_string", srcs = ["byte_string.cc"], diff --git a/common/internal/unknowns.cc b/common/internal/unknowns.cc new file mode 100644 index 000000000..34fc05ed6 --- /dev/null +++ b/common/internal/unknowns.cc @@ -0,0 +1,265 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/internal/unknowns.h" + +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "base/attribute.h" +#include "base/attribute_matcher.h" +#include "google/protobuf/arena.h" + +namespace cel::common_internal { + +namespace { + +template +[[nodiscard]] +size_t PreflightSetUnion(T* const* first1, T* const* last1, T* const* first2, + T* const* last2) { + size_t preflight = 0; + for (; first1 != last1; ++preflight) { + if (first2 == last2) { + return preflight + static_cast(last1 - first1); + } + if (*first2 < *first1) { + ++first2; + } else { + if (!(*first1 < *first2)) { + ++first2; + } + ++first1; + } + } + return preflight + static_cast(last2 - first2); +} + +template +[[nodiscard]] +T** SetUnion(T* const* first1, T* const* last1, T* const* first2, + T* const* last2, T** out) { + for (; first1 != last1; ++out) { + if (first2 == last2) { + return std::copy(first1, last1, out); + } + if (*first2 < *first1) { + *out = *first2++; + } else { + *out = *first1; + if (!(*first1 < *first2)) { + ++first2; + } + ++first1; + } + } + return std::copy(first2, last2, out); +} + +} // namespace + +absl::string_view ArenaString(google::protobuf::Arena* absl_nonnull arena, + absl::string_view string) { + if (string.empty()) { + return ""; + } + char* str = reinterpret_cast(arena->AllocateAligned(string.size())); + std::memcpy(str, string.data(), string.size()); + return absl::string_view(str, string.size()); +} + +UnknownAttributeNode* absl_nonnull UnknownTree::Convert( + const AttributeMatcherNode* absl_nonnull matcher) { + ABSL_DCHECK(!matcher->key().IsWildcard()); + const size_t scratch_size = matcher->depth() + 1; + if (scratch_size > scratch_.size()) { + scratch_.resize(scratch_size); + } + const AttributeMatcherNode** scratch_data = + reinterpret_cast(scratch_.data()); + const AttributeMatcherNode* node = matcher; + for (size_t i = scratch_size; i > 0; --i) { + scratch_data[i - 1] = node; + node = node->parent(); + } + UnknownAttributeNode* unknown = Root()->Attributes()->Step( + this, *scratch_data[0]->key().ToQualifierView()); + for (size_t i = 1; i < scratch_size; ++i) { + unknown = unknown->Step(this, *scratch_data[i]->key().ToQualifierView()); + } + return unknown; +} + +std::string UnknownTree::ToString( + const AttributeMatcherNode* absl_nonnull matcher, + const AttributeQualifierView& key) { + const bool is_wildcard = matcher->key().IsWildcard(); + ABSL_DCHECK(is_wildcard || matcher->key() == key); + if (is_wildcard) { + std::string result = ToString(matcher->parent()); + key.AppendJoinString(&result); + } + return ToString(matcher); +} + +std::string UnknownTree::ToString( + const AttributeMatcherNode* absl_nonnull matcher) { + const size_t scratch_size = matcher->depth() + 1; + if (scratch_size > scratch_.size()) { + scratch_.resize(scratch_size); + } + const AttributeMatcherNode** scratch_data = + reinterpret_cast(scratch_.data()); + const AttributeMatcherNode* node = matcher; + for (size_t i = scratch_size; i > 0; --i) { + scratch_data[i - 1] = node; + node = node->parent(); + } + std::string result = scratch_data[0]->key().GetString(); + for (size_t i = 1; i < scratch_size; ++i) { + scratch_data[i]->key().AppendJoinString(&result); + } + return result; +} + +std::string UnknownTree::ToString( + const UnknownAttributeNode* absl_nonnull unknown) { + const size_t scratch_size = unknown->depth(); + if (scratch_size > scratch_.size()) { + scratch_.resize(scratch_size); + } + const UnknownAttributeNode** scratch_data = + reinterpret_cast(scratch_.data()); + const UnknownAttributeNode* node = unknown; + for (size_t i = scratch_size; i > 0; --i) { + scratch_data[i - 1] = node; + node = node->parent(); + } + std::string result = std::string(scratch_data[0]->key().GetString()); + for (size_t i = 1; i < scratch_size; ++i) { + scratch_data[i]->key().AppendJoinString(&result); + } + return result; +} + +UnknownSet UnknownTree::GraftImpl(const UnknownSet& in) { + ABSL_DCHECK_NE(in.root, RootIfPresent()); + // This should only happen in rare circumstances where somebody passed an + // UnknownValue from one expression evaluation into another. + UnknownAttributeSet* unknown_attribute_set = nullptr; + UnknownFunctionSet* unknown_function_set = nullptr; + if (in.attributes != nullptr && in.attributes->size > 0) { + unknown_attribute_set = + UnknownAttributeSet::Allocate(in.attributes->size, GetArena()); + unknown_attribute_set->max_depth = in.attributes->max_depth; + for (uint32_t i = 0; i < in.attributes->size; ++i) { + unknown_attribute_set->nodes[i] = GraftImpl(in.attributes->nodes[i]); + } + std::sort(unknown_attribute_set->nodes, + unknown_attribute_set->nodes + unknown_attribute_set->size); + } + if (in.functions != nullptr && in.functions->size > 0) { + unknown_function_set = + UnknownFunctionSet::Allocate(in.functions->size, GetArena()); + for (uint32_t i = 0; i < in.functions->size; ++i) { + unknown_function_set->nodes[i] = GraftImpl(in.functions->nodes[i]); + } + std::sort(unknown_function_set->nodes, + unknown_function_set->nodes + unknown_function_set->size); + } + return UnknownSet{ + .root = Root(), + .attributes = unknown_attribute_set, + .functions = unknown_function_set, + }; +} + +UnknownAttributeNode* UnknownTree::GraftImpl(const UnknownAttributeNode* in) { + const size_t scratch_size = in->depth(); + if (scratch_size > scratch_.size()) { + scratch_.resize(scratch_size); + } + const UnknownAttributeNode** scratch_data = + reinterpret_cast(scratch_.data()); + for (size_t i = scratch_size; i > 0; --i) { + scratch_data[i - 1] = in; + in = in->parent(); + } + UnknownAttributeNode* out = Root()->Attributes(); + for (size_t i = 0; i < scratch_size; ++i) { + out = out->Step(this, scratch_data[i]->key()); + } + return out; +} + +const UnknownFunctionSet* absl_nonnull MergeUnknownFunctionSets( + const UnknownFunctionSet* absl_nonnull lhs, + const UnknownFunctionSet* absl_nonnull rhs, + google::protobuf::Arena* absl_nonnull arena) { + UnknownFunctionNode* const* const lhs_begin = lhs->nodes; + const size_t lhs_size = lhs->size; + UnknownFunctionNode* const* const lhs_end = lhs_begin + lhs_size; + UnknownFunctionNode* const* const rhs_begin = rhs->nodes; + const size_t rhs_size = rhs->size; + UnknownFunctionNode* const* const rhs_end = rhs_begin + rhs_size; + const size_t size = + (PreflightSetUnion)(lhs_begin, lhs_end, rhs_begin, rhs_end); + if (lhs_size >= rhs_size && lhs_size == size) { + return lhs; + } + if (rhs_size > lhs_size && rhs_size == size) { + return rhs; + } + UnknownFunctionSet* ufs = UnknownFunctionSet::Allocate(size, arena); + UnknownFunctionNode** out_begin = ufs->nodes; + UnknownFunctionNode** out_end = out_begin + size; + out_begin = (SetUnion)(lhs_begin, lhs_end, rhs_begin, rhs_end, out_begin); + ABSL_DCHECK_EQ(out_begin, out_end); + return ufs; +} + +const UnknownAttributeSet* absl_nonnull MergeUnknownAttributeSets( + const UnknownAttributeSet* absl_nonnull lhs, + const UnknownAttributeSet* absl_nonnull rhs, + google::protobuf::Arena* absl_nonnull arena) { + UnknownAttributeNode* const* const lhs_begin = lhs->nodes; + const size_t lhs_size = lhs->size; + UnknownAttributeNode* const* const lhs_end = lhs_begin + lhs_size; + UnknownAttributeNode* const* const rhs_begin = rhs->nodes; + const size_t rhs_size = rhs->size; + UnknownAttributeNode* const* const rhs_end = rhs_begin + rhs_size; + const size_t size = + (PreflightSetUnion)(lhs_begin, lhs_end, rhs_begin, rhs_end); + if (lhs_size >= rhs_size && lhs_size == size) { + return lhs; + } + if (rhs_size > lhs_size && rhs_size == size) { + return rhs; + } + UnknownAttributeSet* uas = UnknownAttributeSet::Allocate(size, arena); + uas->max_depth = std::max(lhs->max_depth, rhs->max_depth); + UnknownAttributeNode** out_begin = uas->nodes; + UnknownAttributeNode** out_end = out_begin + size; + out_begin = (SetUnion)(lhs_begin, lhs_end, rhs_begin, rhs_end, out_begin); + ABSL_DCHECK_EQ(out_begin, out_end); + return uas; +} + +} // namespace cel::common_internal diff --git a/common/internal/unknowns.h b/common/internal/unknowns.h new file mode 100644 index 000000000..d23517265 --- /dev/null +++ b/common/internal/unknowns.h @@ -0,0 +1,477 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_UNKNOWNS_H_ +#define THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_UNKNOWNS_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/container/flat_hash_set.h" +#include "absl/container/inlined_vector.h" +#include "absl/functional/overload.h" +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "base/attribute.h" +#include "base/attribute_matcher.h" +#include "internal/arena_tree.h" +#include "google/protobuf/arena.h" + +namespace cel::common_internal { + +class UnknownAttributeNode; +class UnknownTree; +class UnknownRoot; +struct UnknownFunctionSet; +struct UnknownAttributeSet; + +// Node in an arena-based red black tree. Leaf nodes are matches to unknown +// attributes. The root node is owned by UnknownRoot. Nodes which are direct +// children of the root node are variables. All nodes below them are qualifiers. +class UnknownAttributeNode + : public internal::ArenaTreeNodeCrtp { + private: + struct Compare { + using is_transparent = void; + + int operator()(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) const { + if (lhs < rhs) { + return -1; + } + if (lhs == rhs) { + return 0; + } + return 1; + } + + int operator()(const AttributeQualifierView& lhs, + const UnknownAttributeNode& rhs) const { + return (*this)(lhs, rhs.key()); + } + + int operator()(const UnknownAttributeNode& lhs, + const AttributeQualifierView& rhs) const { + return (*this)(lhs.key(), rhs); + } + + int operator()(const UnknownAttributeNode& lhs, + const UnknownAttributeNode& rhs) const { + return (*this)(lhs.key(), rhs.key()); + } + }; + + public: + explicit UnknownAttributeNode(UnknownAttributeNode* absl_nullable parent, + const AttributeQualifierView& key) + : ArenaTreeNodeCrtp(), + parent_(parent), + key_(key), + depth_(parent != nullptr ? parent->depth() + 1 : 0) {} + + [[nodiscard]] + const AttributeQualifierView& key() const { + return key_; + } + + [[nodiscard]] + UnknownAttributeNode* parent() const { + return parent_; + } + + [[nodiscard]] + size_t depth() const { + return depth_; + } + + [[nodiscard]] + UnknownAttributeNode* Step(UnknownTree* absl_nonnull tree, + const AttributeQualifierView& key); + + [[nodiscard]] + UnknownAttributeNode* Find(const AttributeQualifierView& key) const { + return internal::ArenaTreeFind(children_, key, Compare{}); + } + + private: + // Pointer to our parent node. If this is null, then this node is the root + // node and its key is invalid. If this is a pointer to the root node then + // this node is a variable and the key is guaranteed to be a string. + UnknownAttributeNode* const absl_nullability_unknown parent_ = nullptr; + const AttributeQualifierView key_; + // How many nodes between us and the root node. + const size_t depth_; + // Root of the red black tree for children. + UnknownAttributeNode* absl_nullable children_ = nullptr; +}; + +// Node in an arena-based red black tree. Leaf nodes are matches to unknown +// functions. The root node is owned by UnknownRoot. +class UnknownFunctionNode + : public internal::ArenaTreeNodeCrtp { + public: + struct Compare { + using is_transparent = void; + + int operator()(absl::string_view lhs, absl::string_view rhs) const { + return lhs.compare(rhs); + } + + int operator()(absl::string_view lhs, + const UnknownFunctionNode& rhs) const { + return (*this)(lhs, rhs.key()); + } + + int operator()(const UnknownFunctionNode& lhs, + absl::string_view rhs) const { + return (*this)(lhs.key(), rhs); + } + + int operator()(const UnknownFunctionNode& lhs, + const UnknownFunctionNode& rhs) const { + return (*this)(lhs.key(), rhs.key()); + } + }; + + explicit UnknownFunctionNode(absl::string_view key) + : ArenaTreeNodeCrtp(), key_(key) {} + + [[nodiscard]] + absl::string_view key() const { + return key_; + } + + private: + const absl::string_view key_; +}; + +class UnknownTree; + +[[nodiscard]] +absl::string_view ArenaString(google::protobuf::Arena* absl_nonnull arena, + absl::string_view string); + +// Holds the the red black tree for functions and the root attribute node which +// holds a red black tree for variables. The root is lazily created on the arena +// by UnknownTree as needed. +class UnknownRoot { + public: + explicit UnknownRoot(google::protobuf::Arena* absl_nonnull arena) + : arena_(arena), attributes_(nullptr, AttributeQualifierView()) {} + + [[nodiscard]] + google::protobuf::Arena* absl_nonnull GetArena() const { + return arena_; + } + + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Attributes() { + return &attributes_; + } + + [[nodiscard]] + const UnknownAttributeNode* absl_nonnull Attributes() const { + return &attributes_; + } + + [[nodiscard]] + UnknownFunctionNode* absl_nonnull SetFunction(UnknownTree* absl_nonnull tree, + absl::string_view name); + + [[nodiscard]] + UnknownFunctionNode* absl_nullable FindFunction(absl::string_view name) const; + + private: + google::protobuf::Arena* const absl_nonnull arena_; + UnknownAttributeNode attributes_; + UnknownFunctionNode* functions_ = nullptr; +}; + +// Represents a set of unknown function nodes. +struct UnknownFunctionSet { + size_t size; + // Nodes are sorted in ascending order by pointer. There are no duplicates. + UnknownFunctionNode* absl_nonnull nodes[]; + + [[nodiscard]] + static UnknownFunctionSet* Allocate(size_t size, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK_GT(size, 0); + UnknownFunctionSet* ptr = reinterpret_cast( + arena->AllocateAligned(offsetof(UnknownFunctionSet, nodes) + + (size * sizeof(UnknownFunctionNode*)))); + ptr->size = size; + return ptr; + } +}; + +// Represents a set of unknown attribute nodes, which themselves represent a +// partial or full match to an unknown attribute. +struct UnknownAttributeSet { + uint32_t size; + uint32_t max_depth; + // Nodes are sorted in ascending order by pointer. There are no duplicates. + UnknownAttributeNode* absl_nonnull nodes[]; + + [[nodiscard]] + static UnknownAttributeSet* Allocate(size_t size, + google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK_GT(size, 0); + ABSL_DCHECK_LE(size, std::numeric_limits::max()); + UnknownAttributeSet* ptr = reinterpret_cast( + arena->AllocateAligned(offsetof(UnknownAttributeSet, nodes) + + (size * sizeof(UnknownAttributeNode*)))); + ptr->size = static_cast(size); + return ptr; + } +}; + +// Underlying representation used by UnknownValue. +struct UnknownSet { + const UnknownRoot* root = nullptr; + const UnknownAttributeSet* attributes = nullptr; + const UnknownFunctionSet* functions = nullptr; +}; + +// Used by the runtime to build unknown attribute and unknown function trees. +class UnknownTree { + public: + UnknownTree() : arena_(nullptr) {} + + UnknownTree(const UnknownTree&) = delete; + UnknownTree& operator=(const UnknownTree&) = delete; + + explicit UnknownTree(google::protobuf::Arena* absl_nonnull arena) : arena_(arena) {} + + void MaybeReset(google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + if (arena_ != arena) { + Reset(arena); + } + } + + void Reset(google::protobuf::Arena* absl_nonnull arena) { + ABSL_DCHECK(arena != nullptr); + arena_ = arena; + root_ = nullptr; + strings_.clear(); + scratch_.clear(); + } + + [[nodiscard]] + google::protobuf::Arena* absl_nonnull GetArena() const { + ABSL_DCHECK(arena_ != nullptr); + return arena_; + } + + [[nodiscard]] + UnknownRoot* absl_nonnull Root() { + UnknownRoot* root = RootIfPresent(); + if (ABSL_PREDICT_FALSE(root == nullptr)) { + root_ = root = google::protobuf::Arena::Create(GetArena(), GetArena()); + } + return root; + } + + [[nodiscard]] + UnknownRoot* absl_nullable RootIfPresent() const { + return root_; + } + + // Called when an unknown attribute match is encountered to materialize the + // match in the unknown attribute tree. + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Convert( + const AttributeMatcherNode* absl_nonnull matcher, + const AttributeQualifierView& key) { + const bool is_wildcard = matcher->key().IsWildcard(); + ABSL_DCHECK(is_wildcard || matcher->key() == key); + if (is_wildcard) { + return Convert(matcher->parent())->Step(this, key); + } + return Convert(matcher); + } + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Convert( + const AttributeMatcherNode* absl_nonnull matcher); + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Convert( + const AttributeMatch& match, const AttributeQualifierView& key) { + return Convert(common_internal::GetAttributeMatcherNode(match), key); + } + [[nodiscard]] + UnknownAttributeNode* absl_nonnull Convert(const AttributeMatch& match) { + return Convert(common_internal::GetAttributeMatcherNode(match)); + } + + // Converts an unknown attribute node to its corresponding attribute pattern + // string, as if the node was converted to AttributePattern and + // AttributePattern::ToString was called. + [[nodiscard]] + std::string ToString(const AttributeMatcherNode* absl_nonnull matcher, + const AttributeQualifierView& key); + [[nodiscard]] + std::string ToString(const AttributeMatch& match, + const AttributeQualifierView& key) { + return ToString(common_internal::GetAttributeMatcherNode(match), key); + } + + // Converts an unknown attribute node to its corresponding attribute string, + // as if the node was converted to Attribute and Attribute::ToString was + // called. + [[nodiscard]] + std::string ToString(const UnknownAttributeNode* absl_nonnull unknown); + + // In the rare event that somebody manages to pass an unknown set from one + // expression into another, we need to be able to graft those nodes into our + // tree. The unknown attribute set and unknown function set sort nodes by + // address for efficiency. This function does that grafting when it is + // required. + [[nodiscard]] + UnknownSet Graft(const UnknownSet& in) { + // Fast path, should be taken almost always. + if (ABSL_PREDICT_TRUE(in.root == nullptr || in.root == RootIfPresent())) { + return in; + } + // Slow path, should be taken extremely rarely. + return GraftImpl(in); + } + + private: + friend class UnknownAttributeNode; + friend class UnknownRoot; + + [[nodiscard]] + UnknownSet GraftImpl(const UnknownSet& in); + + [[nodiscard]] + UnknownAttributeNode* GraftImpl(const UnknownAttributeNode* in); + + [[nodiscard]] + UnknownFunctionNode* GraftImpl(const UnknownFunctionNode* in) { + return Root()->SetFunction(this, in->key()); + } + + [[nodiscard]] + std::string ToString(const AttributeMatcherNode* absl_nonnull matcher); + + [[nodiscard]] + AttributeQualifierView InternKey(const AttributeQualifierView& key) { + return std::visit( + absl::Overload( + [](std::monostate) -> AttributeQualifierView { + return AttributeQualifierView(); + }, + [](bool value) -> AttributeQualifierView { + return AttributeQualifierView::OfBool(value); + }, + [](int64_t value) -> AttributeQualifierView { + return AttributeQualifierView::OfInt(value); + }, + [](uint64_t value) -> AttributeQualifierView { + return AttributeQualifierView::OfUint(value); + }, + [this](absl::string_view value) -> AttributeQualifierView { + return AttributeQualifierView::OfString(InternStringKey(value)); + }), + common_internal::AsVariant(key)); + } + + [[nodiscard]] + absl::string_view InternStringKey(absl::string_view key) { + if (key.empty()) { + return ""; + } + return *strings_.lazy_emplace(key, [this, key](const auto& constructor) { + constructor((ArenaString)(GetArena(), key)); + }); + } + + google::protobuf::Arena* absl_nullability_unknown arena_; + // Created as needed. + UnknownRoot* absl_nullable root_ = nullptr; + // These strings are actually null terminated. + absl::flat_hash_set strings_; + // Temporary storage for materializing AttributeMatcherNode* trail to + // UnknownAttributeNode* or for converting either type to a string. + absl::InlinedVector scratch_; +}; + +inline UnknownAttributeNode* UnknownAttributeNode::Step( + UnknownTree* absl_nonnull tree, const AttributeQualifierView& key) { + auto [node, emplaced] = internal::ArenaTreeLazyEmplace( + tree->GetArena(), &children_, key, Compare{}, + [this, tree, &key](const auto& constructor) -> void { + constructor(this, tree->InternKey(key)); + }); + return node; +} + +inline UnknownFunctionNode* absl_nonnull UnknownRoot::SetFunction( + UnknownTree* absl_nonnull tree, absl::string_view name) { + auto [node, emplaced] = internal::ArenaTreeLazyEmplace( + tree->GetArena(), &functions_, name, UnknownFunctionNode::Compare{}, + [tree, &name](const auto& constructor) -> void { + constructor((ArenaString)(tree->GetArena(), name)); + }); + return node; +} + +inline UnknownFunctionNode* absl_nullable UnknownRoot::FindFunction( + absl::string_view name) const { + return internal::ArenaTreeFind(functions_, name, + UnknownFunctionNode::Compare{}); +} + +[[nodiscard]] +inline const UnknownFunctionSet* absl_nonnull CreateUnknownFunctionSet( + UnknownFunctionNode* absl_nonnull node, google::protobuf::Arena* absl_nonnull arena) { + UnknownFunctionSet* ufs = UnknownFunctionSet::Allocate(1, arena); + ufs->nodes[0] = node; + return ufs; +} + +// Fast two-way unknown function set merger without temporary storage. +[[nodiscard]] +const UnknownFunctionSet* absl_nonnull MergeUnknownFunctionSets( + const UnknownFunctionSet* absl_nonnull lhs, + const UnknownFunctionSet* absl_nonnull rhs, + google::protobuf::Arena* absl_nonnull arena); + +[[nodiscard]] +inline const UnknownAttributeSet* absl_nonnull CreateUnknownAttributeSet( + UnknownAttributeNode* absl_nonnull node, + google::protobuf::Arena* absl_nonnull arena) { + UnknownAttributeSet* uas = UnknownAttributeSet::Allocate(1, arena); + uas->max_depth = static_cast(node->depth()); + uas->nodes[0] = node; + return uas; +} + +// Fast two-way unknown attribute set merger without temporary storage. +[[nodiscard]] +const UnknownAttributeSet* absl_nonnull MergeUnknownAttributeSets( + const UnknownAttributeSet* absl_nonnull lhs, + const UnknownAttributeSet* absl_nonnull rhs, + google::protobuf::Arena* absl_nonnull arena); + +} // namespace cel::common_internal + +#endif // THIRD_PARTY_CEL_CPP_COMMON_INTERNAL_UNKNOWNS_H_ diff --git a/common/internal/unknowns_test.cc b/common/internal/unknowns_test.cc new file mode 100644 index 000000000..5e10364a7 --- /dev/null +++ b/common/internal/unknowns_test.cc @@ -0,0 +1,274 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/internal/unknowns.h" + +#include + +#include "absl/status/status_matchers.h" +#include "base/attribute.h" +#include "base/attribute_matcher.h" +#include "internal/testing.h" +#include "google/protobuf/arena.h" + +namespace cel::common_internal { +namespace { + +using ::absl_testing::IsOk; +using ::testing::IsNull; +using ::testing::NotNull; + +TEST(UnknownTree, MatchToString) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + google::protobuf::Arena arena; + UnknownTree tree(&arena); + EXPECT_EQ(tree.ToString( + matcher.MatchVariable("foo") + .MatchQualifier(AttributeQualifierView::OfString("bar")) + .MatchQualifier(AttributeQualifierView::OfString("baz")), + AttributeQualifierView::OfString("baz")), + "foo.bar.baz"); + EXPECT_EQ(tree.ToString(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeQualifierView::OfString("bar")), + "foo.bar"); + EXPECT_EQ(tree.ToString(matcher.MatchVariable("foo"), + AttributeQualifierView::OfString("foo")), + "foo"); +} + +TEST(UnknownTree, UnknownToString) { + google::protobuf::Arena arena; + UnknownTree tree(&arena); + UnknownAttributeNode* node_foo = tree.Root()->Attributes()->Step( + &tree, AttributeQualifierView::OfString("foo")); + UnknownAttributeNode* node_bar = + node_foo->Step(&tree, AttributeQualifierView::OfString("bar")); + UnknownAttributeNode* node_baz = + node_bar->Step(&tree, AttributeQualifierView::OfString("baz")); + EXPECT_EQ(tree.ToString(node_baz), "foo.bar.baz"); + EXPECT_EQ(tree.ToString(node_bar), "foo.bar"); + EXPECT_EQ(tree.ToString(node_foo), "foo"); +} + +TEST(UnknownTree, UnknownAttributeConvert) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + google::protobuf::Arena arena; + UnknownTree tree(&arena); + UnknownAttributeNode* node_baz = + tree.Convert(matcher.MatchVariable("foo") + .MatchQualifier(AttributeQualifierView::OfString("bar")) + .MatchQualifier(AttributeQualifierView::OfString("baz")), + AttributeQualifierView::OfString("baz")); + UnknownAttributeNode* node_bar = + tree.Convert(matcher.MatchVariable("foo").MatchQualifier( + AttributeQualifierView::OfString("bar")), + AttributeQualifierView::OfString("bar")); + UnknownAttributeNode* node_foo = tree.Convert( + matcher.MatchVariable("foo"), AttributeQualifierView::OfString("foo")); + EXPECT_EQ(node_baz->parent(), node_bar); + EXPECT_EQ(node_bar->parent(), node_foo); + EXPECT_EQ(tree.ToString(node_baz), "foo.bar.baz"); + EXPECT_EQ(tree.ToString(node_bar), "foo.bar"); + EXPECT_EQ(tree.ToString(node_foo), "foo"); + EXPECT_EQ( + tree.Root()->Attributes()->Find(AttributeQualifierView::OfString("foo")), + node_foo); +} + +TEST(UnknownTree, UnknownFunction) { + google::protobuf::Arena arena; + UnknownTree tree(&arena); + + EXPECT_THAT(tree.Root()->FindFunction("foo"), IsNull()); + UnknownFunctionNode* node_foo = tree.Root()->SetFunction(&tree, "foo"); + EXPECT_THAT(node_foo, NotNull()); + EXPECT_EQ(tree.Root()->FindFunction("foo"), node_foo); + + EXPECT_THAT(tree.Root()->FindFunction("bar"), IsNull()); + UnknownFunctionNode* node_bar = tree.Root()->SetFunction(&tree, "bar"); + EXPECT_THAT(node_bar, NotNull()); + EXPECT_NE(node_bar, node_foo); + EXPECT_EQ(tree.Root()->FindFunction("bar"), node_bar); + + EXPECT_THAT(tree.Root()->FindFunction("baz"), IsNull()); + UnknownFunctionNode* node_baz = tree.Root()->SetFunction(&tree, "baz"); + EXPECT_THAT(node_baz, NotNull()); + EXPECT_NE(node_baz, node_foo); + EXPECT_EQ(tree.Root()->FindFunction("baz"), node_baz); +} + +void TestUnknownAttributeSets(const UnknownAttributeSet* lhs, + const UnknownAttributeSet* rhs) { + EXPECT_EQ(lhs->size, rhs->size); + EXPECT_EQ(lhs->max_depth, rhs->max_depth); + EXPECT_TRUE(std::is_sorted(lhs->nodes, lhs->nodes + lhs->size)); + EXPECT_TRUE(std::is_sorted(rhs->nodes, rhs->nodes + rhs->size)); + EXPECT_TRUE(std::equal(lhs->nodes, lhs->nodes + lhs->size, rhs->nodes, + rhs->nodes + rhs->size)); +} + +TEST(UnknownTree, UnknownAttributeSet) { + google::protobuf::Arena arena; + UnknownTree tree(&arena); + UnknownAttributeNode* node_foo = tree.Root()->Attributes()->Step( + &tree, AttributeQualifierView::OfString("foo")); + UnknownAttributeNode* node_bar = tree.Root()->Attributes()->Step( + &tree, AttributeQualifierView::OfString("bar")); + UnknownAttributeNode* node_baz = tree.Root()->Attributes()->Step( + &tree, AttributeQualifierView::OfString("baz")); + + const UnknownAttributeSet* set_foo = + CreateUnknownAttributeSet(node_foo, tree.GetArena()); + EXPECT_EQ(set_foo->size, 1); + EXPECT_EQ(set_foo->max_depth, 1); + EXPECT_EQ(set_foo->nodes[0], node_foo); + TestUnknownAttributeSets(CreateUnknownAttributeSet(node_foo, tree.GetArena()), + set_foo); + + const UnknownAttributeSet* set_bar = + CreateUnknownAttributeSet(node_bar, tree.GetArena()); + EXPECT_EQ(set_bar->size, 1); + EXPECT_EQ(set_bar->max_depth, 1); + EXPECT_EQ(set_bar->nodes[0], node_bar); + TestUnknownAttributeSets(CreateUnknownAttributeSet(node_bar, tree.GetArena()), + set_bar); + + const UnknownAttributeSet* set_baz = + CreateUnknownAttributeSet(node_baz, tree.GetArena()); + EXPECT_EQ(set_baz->size, 1); + EXPECT_EQ(set_baz->max_depth, 1); + EXPECT_EQ(set_baz->nodes[0], node_baz); + TestUnknownAttributeSets(CreateUnknownAttributeSet(node_baz, tree.GetArena()), + set_baz); + + const UnknownAttributeSet* set_foo_bar = + MergeUnknownAttributeSets(set_foo, set_bar, &arena); + EXPECT_EQ(set_foo_bar->size, 2); + EXPECT_EQ(set_foo_bar->max_depth, 1); + EXPECT_TRUE(std::is_sorted(set_foo_bar->nodes, + set_foo_bar->nodes + set_foo_bar->size)); + TestUnknownAttributeSets( + MergeUnknownAttributeSets(set_bar, set_foo, tree.GetArena()), + set_foo_bar); + + const UnknownAttributeSet* set_foo_bar_baz = + MergeUnknownAttributeSets(set_foo_bar, set_baz, &arena); + EXPECT_EQ(set_foo_bar_baz->size, 3); + EXPECT_EQ(set_foo_bar_baz->max_depth, 1); + EXPECT_TRUE(std::is_sorted(set_foo_bar_baz->nodes, + set_foo_bar_baz->nodes + set_foo_bar_baz->size)); + TestUnknownAttributeSets( + MergeUnknownAttributeSets(set_baz, set_foo_bar, tree.GetArena()), + set_foo_bar_baz); + + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar_baz, set_foo, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar_baz, set_bar, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar_baz, set_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar_baz, set_foo_bar, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_bar, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_baz, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownAttributeSets(set_foo_bar, set_foo_bar_baz, &arena), + set_foo_bar_baz); +} + +void TestUnknownFunctionSets(const UnknownFunctionSet* lhs, + const UnknownFunctionSet* rhs) { + EXPECT_EQ(lhs->size, rhs->size); + EXPECT_TRUE(std::is_sorted(lhs->nodes, lhs->nodes + lhs->size)); + EXPECT_TRUE(std::is_sorted(rhs->nodes, rhs->nodes + rhs->size)); + EXPECT_TRUE(std::equal(lhs->nodes, lhs->nodes + lhs->size, rhs->nodes, + rhs->nodes + rhs->size)); +} + +TEST(UnknownTree, UnknownFunctionSet) { + google::protobuf::Arena arena; + UnknownTree tree(&arena); + UnknownFunctionNode* node_foo = tree.Root()->SetFunction(&tree, "foo"); + UnknownFunctionNode* node_bar = tree.Root()->SetFunction(&tree, "bar"); + UnknownFunctionNode* node_baz = tree.Root()->SetFunction(&tree, "baz"); + + const UnknownFunctionSet* set_foo = + CreateUnknownFunctionSet(node_foo, tree.GetArena()); + EXPECT_EQ(set_foo->size, 1); + EXPECT_EQ(set_foo->nodes[0], node_foo); + TestUnknownFunctionSets(CreateUnknownFunctionSet(node_foo, tree.GetArena()), + set_foo); + + const UnknownFunctionSet* set_bar = + CreateUnknownFunctionSet(node_bar, tree.GetArena()); + EXPECT_EQ(set_bar->size, 1); + EXPECT_EQ(set_bar->nodes[0], node_bar); + TestUnknownFunctionSets(CreateUnknownFunctionSet(node_bar, tree.GetArena()), + set_bar); + + const UnknownFunctionSet* set_baz = + CreateUnknownFunctionSet(node_baz, tree.GetArena()); + EXPECT_EQ(set_baz->size, 1); + EXPECT_EQ(set_baz->nodes[0], node_baz); + TestUnknownFunctionSets(CreateUnknownFunctionSet(node_baz, tree.GetArena()), + set_baz); + + const UnknownFunctionSet* set_foo_bar = + MergeUnknownFunctionSets(set_foo, set_bar, &arena); + EXPECT_EQ(set_foo_bar->size, 2); + EXPECT_TRUE(std::is_sorted(set_foo_bar->nodes, + set_foo_bar->nodes + set_foo_bar->size)); + TestUnknownFunctionSets(MergeUnknownFunctionSets(set_bar, set_foo, &arena), + set_foo_bar); + + const UnknownFunctionSet* set_foo_bar_baz = + MergeUnknownFunctionSets(set_foo_bar, set_baz, &arena); + EXPECT_EQ(set_foo_bar_baz->size, 3); + EXPECT_TRUE(std::is_sorted(set_foo_bar_baz->nodes, + set_foo_bar_baz->nodes + set_foo_bar_baz->size)); + TestUnknownFunctionSets( + MergeUnknownFunctionSets(set_baz, set_foo_bar, &arena), set_foo_bar_baz); + + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar_baz, set_foo, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar_baz, set_bar, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar_baz, set_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar_baz, set_foo_bar, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_bar, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_baz, set_foo_bar_baz, &arena), + set_foo_bar_baz); + EXPECT_EQ(MergeUnknownFunctionSets(set_foo_bar, set_foo_bar_baz, &arena), + set_foo_bar_baz); +} + +} // namespace +} // namespace cel::common_internal diff --git a/eval/public/cel_attribute_test.cc b/eval/public/cel_attribute_test.cc index 424dae92f..a2d08d819 100644 --- a/eval/public/cel_attribute_test.cc +++ b/eval/public/cel_attribute_test.cc @@ -17,7 +17,6 @@ namespace { using cel::expr::Expr; -using ::absl_testing::IsOkAndHolds; using ::absl_testing::StatusIs; using ::testing::Eq; using ::testing::IsEmpty; @@ -76,7 +75,7 @@ TEST(CelAttributeQualifierTest, TestUint64Access) { EXPECT_TRUE(qualifier.GetUint64Key().has_value()); EXPECT_THAT(qualifier.GetUint64Key().value(), Eq(1UL)); - EXPECT_THAT(qualifier.ToString(), "1"); + EXPECT_THAT(qualifier.ToString(), "1u"); } TEST(CelAttributeQualifierTest, TestStringAccess) { From 97753a06555f3224d74ef10fda7da3e66cd45d88 Mon Sep 17 00:00:00 2001 From: Justin King Date: Fri, 9 Oct 2026 17:21:36 -0700 Subject: [PATCH 281/350] Improve variable matching performance and add utilities for matching attributes PiperOrigin-RevId: 996851438 --- base/attribute.h | 25 +++++++++ base/attribute_matcher.h | 97 ++++++++++++++++++++++++++++++++-- base/attribute_matcher_test.cc | 28 ++++++++++ base/attribute_test.cc | 17 ++++++ 4 files changed, 163 insertions(+), 4 deletions(-) diff --git a/base/attribute.h b/base/attribute.h index 2229dc495..521391c68 100644 --- a/base/attribute.h +++ b/base/attribute.h @@ -116,6 +116,8 @@ class AttributeQualifier { AttributeQualifier& operator=(const AttributeQualifier&) = default; AttributeQualifier& operator=(AttributeQualifier&&) = default; + explicit AttributeQualifier(const AttributeQualifierView& other); + Kind kind() const; ABSL_DEPRECATE_AND_INLINE() @@ -324,6 +326,10 @@ class AttributeQualifierView { AttributeQualifierView& operator=(AttributeQualifier&&) = delete; + explicit operator bool() const { + return !std::holds_alternative(value_); + } + ABSL_DEPRECATE_AND_INLINE() std::optional GetInt64Key() const { return AsInt(); } @@ -450,6 +456,25 @@ class AttributeQualifierView { Variant value_; }; +inline AttributeQualifier::AttributeQualifier( + const AttributeQualifierView& other) + : value_(std::visit( + absl::Overload([](std::monostate) -> Variant { return Variant(); }, + [](int64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](uint64_t value) -> Variant { + return Variant(std::in_place_type, value); + }, + [](absl::string_view value) -> Variant { + return Variant(std::in_place_type, + value); + }, + [](bool value) -> Variant { + return Variant(std::in_place_type, value); + }), + common_internal::AsVariant(other))) {} + // AttributeQualifierPattern matches a segment in // attribute resolution path. AttributeQualifierPattern is capable of // matching path elements of types string/int64/uint64/bool. diff --git a/base/attribute_matcher.h b/base/attribute_matcher.h index 69174868e..01d60ba85 100644 --- a/base/attribute_matcher.h +++ b/base/attribute_matcher.h @@ -23,6 +23,7 @@ #include #include "absl/base/attributes.h" +#include "absl/base/optimization.h" #include "absl/functional/function_ref.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" @@ -78,6 +79,32 @@ class AttributeMatcherNode { const AttributeMatcherNode& rhs) const { return lhs < rhs.key(); } + + bool operator()(absl::string_view lhs, + const AttributeQualifierPattern& rhs) const { + // Fast path for top-level variable matching, should only be called in + // that context. + ABSL_DCHECK(rhs.IsString()); + return lhs < rhs.GetString(); + } + + bool operator()(const AttributeQualifierPattern& lhs, + absl::string_view rhs) const { + // Fast path for top-level variable matching, should only be called in + // that context. + ABSL_DCHECK(lhs.IsString()); + return lhs.GetString() < rhs; + } + + bool operator()(absl::string_view lhs, + const AttributeMatcherNode& rhs) const { + return (*this)(lhs, rhs.key()); + } + + bool operator()(const AttributeMatcherNode& lhs, + absl::string_view rhs) const { + return (*this)(lhs.key(), rhs); + } }; AttributeMatcherNode(AttributeMatcherNode* parent, @@ -113,7 +140,21 @@ class AttributeMatcherNode { [[nodiscard]] const AttributeMatcherNode* MatchQualifier( - const cel::AttributeQualifierView& qualifier) const { + const AttributeQualifierView& qualifier) const { + if (children_ != nullptr) { + auto it = children_->find(qualifier); + if (it != children_->end()) { + return &*it; + } + } else if (wildcard_ != nullptr) { + return wildcard_.get(); + } + return nullptr; + } + + [[nodiscard]] + const AttributeMatcherNode* MatchQualifier( + const AttributeQualifier& qualifier) const { if (children_ != nullptr) { auto it = children_->find(qualifier); if (it != children_->end()) { @@ -199,6 +240,7 @@ class [[nodiscard]] AttributeMatch { // match should be handled. AttributeMatch MatchQualifier(const AttributeQualifierView& qualifier) const { ABSL_DCHECK(!IsFull()); + ABSL_DCHECK(qualifier); if (IsNone()) { return AttributeMatch(); } @@ -206,11 +248,19 @@ class [[nodiscard]] AttributeMatch { } private: + friend class AttributeMatcher; friend AttributeMatch common_internal::MakeAttributeMatch( const common_internal::AttributeMatcherNode* node); friend const common_internal::AttributeMatcherNode* common_internal::GetAttributeMatcherNode(const AttributeMatch& match); + AttributeMatch NotNoneMatchQualifier( + const AttributeQualifierView& qualifier) const { + ABSL_DCHECK(!IsNone()); + ABSL_DCHECK(qualifier); + return AttributeMatch(node_->MatchQualifier(qualifier)); + } + explicit AttributeMatch(const common_internal::AttributeMatcherNode* node) : node_(node) {} @@ -273,13 +323,52 @@ class AttributeMatcher { AttributeMatch MatchVariable(absl::string_view variable) const ABSL_ATTRIBUTE_LIFETIME_BOUND { - auto it = children_.find(AttributeQualifierView::OfString(variable)); - if (it != children_.end()) { - return common_internal::MakeAttributeMatch(&*it); + if (!variable.empty()) { + auto it = children_.find(variable); + if (it != children_.end()) { + return common_internal::MakeAttributeMatch(&*it); + } } return AttributeMatch(); } + AttributeMatch MatchAttribute( + absl::string_view variable, + absl::Span qualifiers) const { + AttributeMatch match = MatchVariable(variable); + for (const auto& qualifier : qualifiers) { + if (ABSL_PREDICT_FALSE(!qualifier)) { + return AttributeMatch(); + } + if (match.IsNone()) { + break; + } + match = match.NotNoneMatchQualifier(qualifier); + } + return match; + } + + AttributeMatch MatchAttribute( + absl::string_view variable, + absl::Span qualifiers) const { + AttributeMatch match = MatchVariable(variable); + for (const auto& qualifier : qualifiers) { + if (ABSL_PREDICT_FALSE(!qualifier)) { + return AttributeMatch(); + } + if (match.IsNone()) { + break; + } + match = match.NotNoneMatchQualifier(qualifier); + } + return match; + } + + AttributeMatch MatchAttribute(const Attribute& attribute) const { + return MatchAttribute(attribute.variable_name(), + attribute.qualifier_path()); + } + // Returns an array of attribute patterns corresponding to the // previously added attribute patterns. The order of the attribute // patterns in the array is not guaranteed. diff --git a/base/attribute_matcher_test.cc b/base/attribute_matcher_test.cc index 899f2f03f..32f002e7e 100644 --- a/base/attribute_matcher_test.cc +++ b/base/attribute_matcher_test.cc @@ -423,5 +423,33 @@ TEST(AttributeMatcher, PastFullMatch) { _); } +TEST(AttributeMatcher, MatchAttribute) { + AttributeMatcher matcher; + EXPECT_THAT(matcher.InsertAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + IsOk()); + EXPECT_THAT(matcher.MatchAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar"), + AttributeQualifier::OfString("baz")})), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT( + matcher.MatchAttribute("foo", {AttributeQualifierView::OfString("bar"), + AttributeQualifierView::OfString("baz")}), + AttributeMatchIs(MatchType::FULL)); + EXPECT_THAT(matcher.MatchAttribute( + Attribute("foo", {AttributeQualifier::OfString("bar")})), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT( + matcher.MatchAttribute("foo", {AttributeQualifierView::OfString("bar")}), + AttributeMatchIs(MatchType::PARTIAL)); + EXPECT_THAT(matcher.MatchAttribute( + Attribute("bar", {AttributeQualifier::OfString("baz")})), + AttributeMatchIs(MatchType::NONE)); + EXPECT_THAT( + matcher.MatchAttribute("bar", {AttributeQualifierView::OfString("baz")}), + AttributeMatchIs(MatchType::NONE)); +} + } // namespace } // namespace cel diff --git a/base/attribute_test.cc b/base/attribute_test.cc index 945a8569a..d9aa1ef98 100644 --- a/base/attribute_test.cc +++ b/base/attribute_test.cc @@ -148,5 +148,22 @@ TEST(AttributeQualifierView, TransparentOrder) { AttributeQualifierView::OfBool(true)); } +template +void TestAttributeQualifierConversion() { + EXPECT_EQ(To(From::OfInt(1)), To::OfInt(1)); + EXPECT_EQ(To(From::OfUint(1)), To::OfUint(1)); + EXPECT_EQ(To(From::OfString("foo")), To::OfString("foo")); + EXPECT_EQ(To(From::OfBool(true)), To::OfBool(true)); +} + +TEST(AttributeQualifier, Conversion) { + TestAttributeQualifierConversion(); + TestAttributeQualifierConversion(); + TestAttributeQualifierConversion(); +} + } // namespace } // namespace cel From ad4a0916ed9aac7ae9b9a31ad68c7b169bb80d7d Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Fri, 9 Oct 2026 19:32:42 -0700 Subject: [PATCH 282/350] No public description PiperOrigin-RevId: 996895698 --- eval/tests/BUILD | 1 + eval/tests/benchmark_test.cc | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eval/tests/BUILD b/eval/tests/BUILD index 47d693e17..78a08ed0c 100644 --- a/eval/tests/BUILD +++ b/eval/tests/BUILD @@ -50,6 +50,7 @@ cc_test( "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/status:status_matchers", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto", "@com_google_googleapis//google/rpc/context:attribute_context_cc_proto", "@com_google_protobuf//:protobuf", diff --git a/eval/tests/benchmark_test.cc b/eval/tests/benchmark_test.cc index ff38ed2af..67ab9cd8b 100644 --- a/eval/tests/benchmark_test.cc +++ b/eval/tests/benchmark_test.cc @@ -14,6 +14,7 @@ #include "absl/flags/flag.h" #include "absl/status/status_matchers.h" #include "absl/strings/match.h" +#include "absl/types/span.h" #include "common/ast_proto.h" #include "common/decl.h" #include "common/type.h" @@ -575,9 +576,7 @@ void BM_HasMap(benchmark::State& state) { std::vector> map_pairs{ {CelValue::CreateStringView("path"), CelValue::CreateStringView("path")}}; - auto cel_map = - CreateContainerBackedMap(absl::Span>( - map_pairs.data(), map_pairs.size())); + auto cel_map = CreateContainerBackedMap(absl::MakeSpan(map_pairs)); activation.InsertValue("request", CelValue::CreateMap((*cel_map).get())); for (auto _ : state) { From 5c5f381af346b2dd6794edc0983fea6bbd5d5733 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sat, 10 Oct 2026 05:00:52 -0700 Subject: [PATCH 283/350] No public description PiperOrigin-RevId: 997060290 --- common/internal/byte_string.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/internal/byte_string.cc b/common/internal/byte_string.cc index cbf28f422..dcc8479ca 100644 --- a/common/internal/byte_string.cc +++ b/common/internal/byte_string.cc @@ -133,7 +133,7 @@ absl::optional ByteString::TryFlat() const { if (auto flat = rep_.large.data->TryFlat(); flat.has_value()) { return flat->substr(rep_.large.offset, rep_.large.size); } - return absl::nullopt; + return std::nullopt; } } } From d2c67ef1c5f7d692684f1ca7b7d5663b1426170b Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 11 Oct 2026 19:42:03 -0700 Subject: [PATCH 284/350] No public description PiperOrigin-RevId: 997630072 --- parser/internal/antlr_parser.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/internal/antlr_parser.cc b/parser/internal/antlr_parser.cc index cb9e63258..14f125c84 100644 --- a/parser/internal/antlr_parser.cc +++ b/parser/internal/antlr_parser.cc @@ -725,7 +725,7 @@ class ParserVisitor final : public CelBaseVisitor, }; template ::value>> + std::is_base_of_v>> T* tree_as(antlr4::tree::ParseTree* tree) { return dynamic_cast(tree); } From a15d985696dd3ca1e28d50c5125e0de17bfd2549 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 11 Oct 2026 19:47:24 -0700 Subject: [PATCH 285/350] No public description PiperOrigin-RevId: 997631942 --- eval/public/ast_rewrite_test.cc | 4 ++-- eval/public/builtin_func_test.cc | 4 ++-- eval/public/extension_func_test.cc | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eval/public/ast_rewrite_test.cc b/eval/public/ast_rewrite_test.cc index b2ee8d13c..c7297b5b1 100644 --- a/eval/public/ast_rewrite_test.cc +++ b/eval/public/ast_rewrite_test.cc @@ -501,7 +501,7 @@ TEST(AstCrawlerTest, CheckExprRewriteHandlers) { // Simple rewrite that replaces a select path with a dot-qualified identifier. class RewriterExample : public AstRewriterBase { public: - RewriterExample() {} + RewriterExample() = default; bool PostVisitRewrite(Expr* expr, const SourcePosition* info) override { if (target_.has_value() && expr->id() == *target_) { expr->mutable_ident_expr()->set_name("com.google.Identifier"); @@ -552,7 +552,7 @@ TEST(AstRewrite, SelectRewriteExample) { // both passes. class PreRewriterExample : public AstRewriterBase { public: - PreRewriterExample() {} + PreRewriterExample() = default; bool PreVisitRewrite(Expr* expr, const SourcePosition* info) override { if (expr->ident_expr().name() == "x") { expr->mutable_ident_expr()->set_name("y"); diff --git a/eval/public/builtin_func_test.cc b/eval/public/builtin_func_test.cc index a69091de4..6355bd033 100644 --- a/eval/public/builtin_func_test.cc +++ b/eval/public/builtin_func_test.cc @@ -55,7 +55,7 @@ using ::testing::Eq; class BuiltinsTest : public ::testing::Test { protected: - BuiltinsTest() {} + BuiltinsTest() = default; // Helper method. Looks up in registry and tests comparison operation. void PerformRun(absl::string_view operation, absl::optional target, @@ -1123,7 +1123,7 @@ class FakeList : public CelList { class FakeErrorMap : public CelMap { public: - FakeErrorMap() {} + FakeErrorMap() = default; int size() const override { return 0; } diff --git a/eval/public/extension_func_test.cc b/eval/public/extension_func_test.cc index 2e2497d7d..c9f5fb4fa 100644 --- a/eval/public/extension_func_test.cc +++ b/eval/public/extension_func_test.cc @@ -32,7 +32,7 @@ static const int kNanosPerSecond = 1000000000; class ExtensionTest : public ::testing::Test { protected: - ExtensionTest() {} + ExtensionTest() = default; void SetUp() override { ASSERT_OK(RegisterBuiltinFunctions(®istry_)); From 6d99aee9bbbd254c0d64cbe3bfbec05342998cf6 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 11 Oct 2026 20:16:48 -0700 Subject: [PATCH 286/350] No public description PiperOrigin-RevId: 997643062 --- extensions/formatting.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/formatting.cc b/extensions/formatting.cc index c222be1ad..d05fbe7f6 100644 --- a/extensions/formatting.cc +++ b/extensions/formatting.cc @@ -296,7 +296,7 @@ absl::StatusOr FormatBinary( sign_bit = true; // Negating min int is undefined behavior, so we need to use unsigned // arithmetic. - using unsigned_type = std::make_unsigned::type; + using unsigned_type = std::make_unsigned_t; unsigned_value = -static_cast(tmp); } else { unsigned_value = tmp; @@ -354,7 +354,7 @@ absl::StatusOr FormatHex( if (tmp < 0) { // Negating min int is undefined behavior, so we need to use unsigned // arithmetic. - using unsigned_type = std::make_unsigned::type; + using unsigned_type = std::make_unsigned_t; scratch = absl::StrFormat("-%x", -static_cast(tmp)); } else { scratch = absl::StrFormat("%x", tmp); @@ -386,7 +386,7 @@ absl::StatusOr FormatOctal( if (tmp < 0) { // Negating min int is undefined behavior, so we need to use unsigned // arithmetic. - using unsigned_type = std::make_unsigned::type; + using unsigned_type = std::make_unsigned_t; scratch = absl::StrFormat("-%o", -static_cast(tmp)); } else { scratch = absl::StrFormat("%o", tmp); From 0e9ae4e3eaaf960fe66214bc48b2423f8ca98f1c Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Sun, 11 Oct 2026 20:20:25 -0700 Subject: [PATCH 287/350] No public description PiperOrigin-RevId: 997644321 --- common/ast_rewrite_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/ast_rewrite_test.cc b/common/ast_rewrite_test.cc index 5417b23ac..748bd2473 100644 --- a/common/ast_rewrite_test.cc +++ b/common/ast_rewrite_test.cc @@ -495,7 +495,7 @@ TEST(AstCrawlerTest, CheckExprRewriteHandlers) { // Simple rewrite that replaces a select path with a dot-qualified identifier. class RewriterExample : public AstRewriterBase { public: - RewriterExample() {} + RewriterExample() = default; bool PostVisitRewrite(Expr& expr) override { if (target_.has_value() && expr.id() == *target_) { expr.mutable_ident_expr().set_name("com.google.Identifier"); @@ -554,7 +554,7 @@ TEST(AstRewrite, SelectRewriteExample) { // both passes. class PreRewriterExample : public AstRewriterBase { public: - PreRewriterExample() {} + PreRewriterExample() = default; bool PreVisitRewrite(Expr& expr) override { if (expr.ident_expr().name() == "x") { expr.mutable_ident_expr().set_name("y"); From 86f872117d0f689dd7ea6d71485d9f3f724aae10 Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 12 Oct 2026 09:27:10 -0700 Subject: [PATCH 288/350] Add some unknown value testing with the modern API PiperOrigin-RevId: 997967435 --- eval/eval/BUILD | 10 +- eval/eval/create_map_step_test.cc | 54 --- eval/eval/create_struct_step_test.cc | 28 -- eval/eval/function_step_test.cc | 11 +- eval/eval/logic_step_test.cc | 116 ------ eval/eval/select_step_test.cc | 20 -- eval/eval/ternary_step_test.cc | 68 ---- eval/tests/BUILD | 33 ++ eval/tests/modern_unknown_test.cc | 503 +++++++++++++++++++++++++++ 9 files changed, 544 insertions(+), 299 deletions(-) create mode 100644 eval/tests/modern_unknown_test.cc diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 37b88db94..13efe4c0e 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -550,6 +550,7 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":evaluator_core", + "//base:attributes", "//base:data", "//common:constant", "//common:expr", @@ -587,14 +588,10 @@ cc_test( deps = [ ":cel_expression_flat_impl", ":evaluator_core", - "//base:attributes", "//base:data", "//common:expr", "//eval/public:activation", - "//eval/public:cel_attribute", "//eval/public:cel_value", - "//eval/public:unknown_attribute_set", - "//eval/public:unknown_set", "//internal:status_macros", "//internal:testing", "//runtime:runtime_options", @@ -856,14 +853,9 @@ cc_test( ":cel_expression_flat_impl", ":evaluator_core", ":ternary_step", - "//base:attributes", "//base:data", - "//common:expr", "//eval/public:activation", - "//eval/public:cel_attribute", "//eval/public:cel_value", - "//eval/public:unknown_attribute_set", - "//eval/public:unknown_set", "//internal:testing", "//runtime:runtime_options", "//runtime/internal:runtime_env", diff --git a/eval/eval/create_map_step_test.cc b/eval/eval/create_map_step_test.cc index 163fed860..f9f11516c 100644 --- a/eval/eval/create_map_step_test.cc +++ b/eval/eval/create_map_step_test.cc @@ -136,25 +136,6 @@ TEST_P(CreateMapStepTest, TestCreateEmptyMap) { ASSERT_EQ(cel_map->size(), 0); } -// Test message creation if unknown argument is passed -TEST(CreateMapStepTest, TestMapCreateWithUnknown) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - UnknownSet unknown_set; - std::vector> entries; - - std::vector kKeys = {"test2", "test1"}; - - entries.push_back( - {CelValue::CreateString(&kKeys[0]), CelValue::CreateInt64(2)}); - entries.push_back({CelValue::CreateString(&kKeys[1]), - CelValue::CreateUnknownSet(&unknown_set)}); - - ASSERT_OK_AND_ASSIGN(CelValue result, RunCreateMapExpression( - env, entries, &arena, true, false)); - ASSERT_TRUE(result.IsUnknownSet()); -} - TEST(CreateMapStepTest, TestMapCreateWithError) { absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); Arena arena; @@ -172,41 +153,6 @@ TEST(CreateMapStepTest, TestMapCreateWithError) { EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kCancelled)); } -TEST(CreateMapStepTest, TestMapCreateWithErrorRecursiveProgram) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - UnknownSet unknown_set; - absl::Status error = absl::CancelledError(); - std::vector> entries; - entries.push_back({CelValue::CreateStringView("foo"), - CelValue::CreateUnknownSet(&unknown_set)}); - entries.push_back( - {CelValue::CreateStringView("bar"), CelValue::CreateError(&error)}); - - ASSERT_OK_AND_ASSIGN(CelValue result, RunCreateMapExpression( - env, entries, &arena, true, true)); - ASSERT_TRUE(result.IsError()); - EXPECT_THAT(*result.ErrorOrDie(), StatusIs(absl::StatusCode::kCancelled)); -} - -TEST(CreateMapStepTest, TestMapCreateWithUnknownRecursiveProgram) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - UnknownSet unknown_set; - std::vector> entries; - - std::vector kKeys = {"test2", "test1"}; - - entries.push_back( - {CelValue::CreateString(&kKeys[0]), CelValue::CreateInt64(2)}); - entries.push_back({CelValue::CreateString(&kKeys[1]), - CelValue::CreateUnknownSet(&unknown_set)}); - - ASSERT_OK_AND_ASSIGN(CelValue result, RunCreateMapExpression( - env, entries, &arena, true, true)); - ASSERT_TRUE(result.IsUnknownSet()); -} - // Test that String Map is created successfully. TEST_P(CreateMapStepTest, TestCreateStringMap) { Arena arena; diff --git a/eval/eval/create_struct_step_test.cc b/eval/eval/create_struct_step_test.cc index e348ecd20..c6fa6f036 100644 --- a/eval/eval/create_struct_step_test.cc +++ b/eval/eval/create_struct_step_test.cc @@ -229,34 +229,6 @@ TEST(CreateCreateStructStepTest, TestMessageCreateErrorRecursive) { StatusIs(absl::StatusCode::kCancelled)); } -// Test message creation if unknown argument is passed -TEST(CreateCreateStructStepTest, TestMessageCreateWithUnknown) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - TestMessage test_msg; - UnknownSet unknown_set; - - auto eval_status = - RunExpression(env, "bool_value", CelValue::CreateUnknownSet(&unknown_set), - &arena, true, /*enable_recursive_planning=*/false); - ASSERT_THAT(eval_status, IsOk()); - ASSERT_TRUE(eval_status->IsUnknownSet()); -} - -// Test message creation if unknown argument is passed -TEST(CreateCreateStructStepTest, TestMessageCreateWithUnknownRecursive) { - absl_nonnull std::shared_ptr env = NewTestingRuntimeEnv(); - Arena arena; - TestMessage test_msg; - UnknownSet unknown_set; - - auto eval_status = - RunExpression(env, "bool_value", CelValue::CreateUnknownSet(&unknown_set), - &arena, true, /*enable_recursive_planning=*/true); - ASSERT_THAT(eval_status, IsOk()); - ASSERT_TRUE(eval_status->IsUnknownSet()) << eval_status->DebugString(); -} - // Test that fields of type bool are set correctly TEST_P(CreateCreateStructStepTest, TestSetBoolField) { TestMessage test_msg; diff --git a/eval/eval/function_step_test.cc b/eval/eval/function_step_test.cc index 9cb338af2..9b9ce6a36 100644 --- a/eval/eval/function_step_test.cc +++ b/eval/eval/function_step_test.cc @@ -12,6 +12,8 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" +#include "base/attribute.h" +#include "base/attribute_set.h" #include "base/type_provider.h" #include "common/constant.h" #include "common/expr.h" @@ -162,7 +164,8 @@ class SinkFunction : public CelFunction { // Create and initialize a registry with some default functions. void AddDefaults(CelFunctionRegistry& registry) { - static UnknownSet* unknown_set = new UnknownSet(); + static UnknownSet* unknown_set = + new UnknownSet(cel::AttributeSet({cel::Attribute("foo")})); EXPECT_TRUE(registry .Register(std::make_unique( CelValue::CreateInt64(3), "Const3")) @@ -863,7 +866,7 @@ TEST(FunctionStepTestUnknownFunctionResults, UnknownVsErrorPrecedenceTest) { CelError error0 = absl::CancelledError(); CelValue error_value = CelValue::CreateError(&error0); - UnknownSet unknown_set; + UnknownSet unknown_set(cel::AttributeSet({cel::Attribute("foo")})); CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); ASSERT_OK(registry.Register( @@ -960,7 +963,7 @@ class NullFunction : public CelFunction { TEST(FunctionStepStrictnessTest, IfFunctionStrictAndGivenUnknownSkipsInvocation) { - UnknownSet unknown_set; + UnknownSet unknown_set(cel::AttributeSet({cel::Attribute("foo")})); CelFunctionRegistry registry; ASSERT_THAT(registry.Register(std::make_unique( CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown")), @@ -991,7 +994,7 @@ TEST(FunctionStepStrictnessTest, } TEST(FunctionStepStrictnessTest, IfFunctionNonStrictAndGivenUnknownInvokesIt) { - UnknownSet unknown_set; + UnknownSet unknown_set(cel::AttributeSet({cel::Attribute("foo")})); CelFunctionRegistry registry; ASSERT_THAT(registry.Register(std::make_unique( CelValue::CreateUnknownSet(&unknown_set), "ConstUnknown")), diff --git a/eval/eval/logic_step_test.cc b/eval/eval/logic_step_test.cc index e82c4b7cf..59888b20a 100644 --- a/eval/eval/logic_step_test.cc +++ b/eval/eval/logic_step_test.cc @@ -3,23 +3,17 @@ #include #include #include -#include #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/status_matchers.h" #include "absl/strings/string_view.h" -#include "base/attribute.h" -#include "base/attribute_set.h" #include "base/type_provider.h" #include "common/expr.h" #include "eval/eval/cel_expression_flat_impl.h" #include "eval/eval/evaluator_core.h" #include "eval/public/activation.h" -#include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" -#include "eval/public/unknown_attribute_set.h" -#include "eval/public/unknown_set.h" #include "internal/status_macros.h" #include "internal/testing.h" #include "runtime/internal/runtime_env.h" @@ -32,14 +26,11 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::cel::Attribute; -using ::cel::AttributeSet; using ::cel::Expr; using ::cel::TypeProvider; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::google::protobuf::Arena; -using ::testing::Eq; class LogicStepTest : public testing::TestWithParam { public: @@ -193,113 +184,6 @@ TEST_P(LogicStepTest, TestOrLogicErrorHandling) { ASSERT_TRUE(result.BoolOrDie()); } -TEST_F(LogicStepTest, TestAndLogicUnknownHandling) { - CelValue result; - UnknownSet unknown_set; - CelError cel_error = absl::CancelledError(); - CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); - CelValue error_value = CelValue::CreateError(&cel_error); - absl::Status status = EvaluateLogic(unknown_value, CelValue::CreateBool(true), - false, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(CelValue::CreateBool(true), unknown_value, false, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(CelValue::CreateBool(false), unknown_value, false, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_FALSE(result.BoolOrDie()); - - status = EvaluateLogic(unknown_value, CelValue::CreateBool(false), false, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_FALSE(result.BoolOrDie()); - - status = EvaluateLogic(error_value, unknown_value, false, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(unknown_value, error_value, false, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - CelAttribute attr0("name0", {}), attr1("name1", {}); - UnknownAttributeSet unknown_attr_set0({attr0}); - UnknownAttributeSet unknown_attr_set1({attr1}); - UnknownSet unknown_set0(unknown_attr_set0); - UnknownSet unknown_set1(unknown_attr_set1); - - EXPECT_THAT(unknown_attr_set0.size(), Eq(1)); - EXPECT_THAT(unknown_attr_set1.size(), Eq(1)); - - status = EvaluateLogic(CelValue::CreateUnknownSet(&unknown_set0), - CelValue::CreateUnknownSet(&unknown_set1), false, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - ASSERT_THAT(result.UnknownSetOrDie()->unknown_attributes().size(), Eq(2)); -} - -TEST_F(LogicStepTest, TestOrLogicUnknownHandling) { - CelValue result; - UnknownSet unknown_set; - CelError cel_error = absl::CancelledError(); - CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); - CelValue error_value = CelValue::CreateError(&cel_error); - absl::Status status = EvaluateLogic( - unknown_value, CelValue::CreateBool(false), true, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(CelValue::CreateBool(false), unknown_value, true, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(CelValue::CreateBool(true), unknown_value, true, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_TRUE(result.BoolOrDie()); - - status = EvaluateLogic(unknown_value, CelValue::CreateBool(true), true, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_TRUE(result.BoolOrDie()); - - status = EvaluateLogic(unknown_value, error_value, true, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - status = EvaluateLogic(error_value, unknown_value, true, &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - CelAttribute attr0("name0", {}), attr1("name1", {}); - UnknownAttributeSet unknown_attr_set0({attr0}); - UnknownAttributeSet unknown_attr_set1({attr1}); - - UnknownSet unknown_set0(unknown_attr_set0); - UnknownSet unknown_set1(unknown_attr_set1); - - EXPECT_THAT(unknown_attr_set0.size(), Eq(1)); - EXPECT_THAT(unknown_attr_set1.size(), Eq(1)); - - status = EvaluateLogic(CelValue::CreateUnknownSet(&unknown_set0), - CelValue::CreateUnknownSet(&unknown_set1), true, - &result, true); - ASSERT_THAT(status, IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - ASSERT_THAT(result.UnknownSetOrDie()->unknown_attributes().size(), Eq(2)); -} - INSTANTIATE_TEST_SUITE_P(LogicStepTest, LogicStepTest, testing::Bool()); } // namespace diff --git a/eval/eval/select_step_test.cc b/eval/eval/select_step_test.cc index 68c6f6f50..4963550b4 100644 --- a/eval/eval/select_step_test.cc +++ b/eval/eval/select_step_test.cc @@ -290,26 +290,6 @@ TEST_F(SelectStepTest, MapPresenseIsErrorTest) { EXPECT_EQ(result.ErrorOrDie()->code(), absl::StatusCode::kInvalidArgument); } -TEST_F(SelectStepTest, MapPresenseIsTrueWithUnknownTest) { - UnknownSet unknown_set; - std::string key1 = "key1"; - std::vector> key_values{ - {CelValue::CreateString(&key1), - CelValue::CreateUnknownSet(&unknown_set)}}; - - auto map_value = CreateContainerBackedMap( - absl::Span>(key_values)) - .value(); - - RunExpressionOptions options; - options.enable_unknowns = true; - - ASSERT_OK_AND_ASSIGN(CelValue result, - RunExpression(map_value.get(), "key1", true, options)); - ASSERT_TRUE(result.IsBool()); - EXPECT_EQ(result.BoolOrDie(), true); -} - TEST_P(SelectStepConformanceTest, FieldIsNotPresentInProtoTest) { TestMessage message; diff --git a/eval/eval/ternary_step_test.cc b/eval/eval/ternary_step_test.cc index 5d5ac6ae4..d1011f247 100644 --- a/eval/eval/ternary_step_test.cc +++ b/eval/eval/ternary_step_test.cc @@ -3,20 +3,14 @@ #include #include #include -#include #include "absl/base/nullability.h" #include "absl/status/status.h" -#include "base/attribute_set.h" #include "base/type_provider.h" -#include "common/expr.h" #include "eval/eval/cel_expression_flat_impl.h" #include "eval/eval/evaluator_core.h" #include "eval/public/activation.h" -#include "eval/public/cel_attribute.h" #include "eval/public/cel_value.h" -#include "eval/public/unknown_attribute_set.h" -#include "eval/public/unknown_set.h" #include "internal/testing.h" #include "runtime/internal/runtime_env.h" #include "runtime/internal/runtime_env_testing.h" @@ -28,13 +22,11 @@ namespace google::api::expr::runtime { namespace { using ::absl_testing::IsOk; -using ::cel::Expr; using ::cel::RuntimeOptions; using ::cel::TypeProvider; using ::cel::runtime_internal::NewTestingRuntimeEnv; using ::cel::runtime_internal::RuntimeEnv; using ::google::protobuf::Arena; -using ::testing::Eq; class LogicStepTest : public testing::TestWithParam { public: @@ -115,66 +107,6 @@ TEST_P(LogicStepTest, TestErrorHandling) { ASSERT_FALSE(result.BoolOrDie()); } -TEST_F(LogicStepTest, TestUnknownHandling) { - CelValue result; - UnknownSet unknown_set; - CelError cel_error = absl::CancelledError(); - CelValue unknown_value = CelValue::CreateUnknownSet(&unknown_set); - CelValue error_value = CelValue::CreateError(&cel_error); - ASSERT_THAT(EvaluateLogic(unknown_value, CelValue::CreateBool(true), - CelValue::CreateBool(false), &result, true), - IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - ASSERT_THAT(EvaluateLogic(CelValue::CreateBool(true), unknown_value, - CelValue::CreateBool(false), &result, true), - IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - ASSERT_THAT(EvaluateLogic(CelValue::CreateBool(false), unknown_value, - CelValue::CreateBool(false), &result, true), - IsOk()); - ASSERT_TRUE(result.IsBool()); - ASSERT_FALSE(result.BoolOrDie()); - - ASSERT_THAT(EvaluateLogic(error_value, unknown_value, - CelValue::CreateBool(false), &result, true), - IsOk()); - ASSERT_TRUE(result.IsError()); - - ASSERT_THAT(EvaluateLogic(unknown_value, error_value, - CelValue::CreateBool(false), &result, true), - IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - - Expr expr0; - auto& ident_expr0 = expr0.mutable_ident_expr(); - ident_expr0.set_name("name0"); - - Expr expr1; - auto& ident_expr1 = expr1.mutable_ident_expr(); - ident_expr1.set_name("name1"); - - CelAttribute attr0(expr0.ident_expr().name(), {}), - attr1(expr1.ident_expr().name(), {}); - UnknownAttributeSet unknown_attr_set0({attr0}); - UnknownAttributeSet unknown_attr_set1({attr1}); - UnknownSet unknown_set0(unknown_attr_set0); - UnknownSet unknown_set1(unknown_attr_set1); - - EXPECT_THAT(unknown_attr_set0.size(), Eq(1)); - EXPECT_THAT(unknown_attr_set1.size(), Eq(1)); - - ASSERT_THAT(EvaluateLogic(CelValue::CreateUnknownSet(&unknown_set0), - CelValue::CreateUnknownSet(&unknown_set1), - CelValue::CreateBool(false), &result, true), - IsOk()); - ASSERT_TRUE(result.IsUnknownSet()); - const auto& attrs = result.UnknownSetOrDie()->unknown_attributes(); - ASSERT_THAT(attrs, testing::SizeIs(1)); - EXPECT_THAT(attrs.begin()->variable_name(), Eq("name0")); -} - INSTANTIATE_TEST_SUITE_P(LogicStepTest, LogicStepTest, testing::Bool()); } // namespace diff --git a/eval/tests/BUILD b/eval/tests/BUILD index 78a08ed0c..a9f081db0 100644 --- a/eval/tests/BUILD +++ b/eval/tests/BUILD @@ -110,6 +110,39 @@ cc_test( ], ) +cc_test( + name = "modern_unknown_test", + srcs = [ + "modern_unknown_test.cc", + ], + deps = [ + "//base:attributes", + "//common:value", + "//extensions:bindings_ext", + "//extensions:comprehensions_v2_macros", + "//internal:testing", + "//internal:testing_descriptor_pool", + "//internal:testing_message_factory", + "//parser", + "//parser:options", + "//parser:parser_interface", + "//runtime", + "//runtime:activation", + "//runtime:constant_folding", + "//runtime:optional_types", + "//runtime:reference_resolver", + "//runtime:runtime_options", + "//runtime:standard_runtime_builder_factory", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:struct_cc_proto", + ], +) + cc_test( name = "allocation_benchmark_test", size = "small", diff --git a/eval/tests/modern_unknown_test.cc b/eval/tests/modern_unknown_test.cc new file mode 100644 index 000000000..6575e45f3 --- /dev/null +++ b/eval/tests/modern_unknown_test.cc @@ -0,0 +1,503 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "google/protobuf/struct.pb.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "base/attribute.h" +#include "common/value.h" +#include "extensions/bindings_ext.h" +#include "extensions/comprehensions_v2_macros.h" +#include "internal/testing.h" +#include "internal/testing_descriptor_pool.h" +#include "internal/testing_message_factory.h" +#include "parser/options.h" +#include "parser/parser.h" +#include "parser/parser_interface.h" +#include "runtime/activation.h" +#include "runtime/constant_folding.h" +#include "runtime/optional_types.h" +#include "runtime/reference_resolver.h" +#include "runtime/runtime.h" +#include "runtime/runtime_options.h" +#include "runtime/standard_runtime_builder_factory.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/json/json.h" +#include "google/protobuf/text_format.h" + +namespace { + +using absl_testing::IsOk; +using ::testing::UnorderedElementsAre; + +class ModernUnknownTest : public ::testing::Test { + public: + void SetUp() override { + auto parser_builder = cel::NewParserBuilder(cel::ParserOptions{ + .enable_optional_syntax = true, + }); + ABSL_CHECK_OK(cel::extensions::RegisterComprehensionsV2Macros( // Crash OK + *parser_builder)); + for (const auto& macro : cel::extensions::bindings_macros()) { + ABSL_CHECK_OK(parser_builder->AddMacro(macro)); // Crash OK + } + auto status_or_parser = parser_builder->Build(); + ABSL_CHECK_OK(status_or_parser); // Crash OK + parser_ = *std::move(status_or_parser); + auto status_or_runtime_builder = cel::CreateStandardRuntimeBuilder( + cel::internal::GetTestingDescriptorPool(), + cel::RuntimeOptions{ + .unknown_processing = + cel::UnknownProcessingOptions::kAttributeAndFunction, + .enable_missing_attribute_errors = true, + .enable_qualified_type_identifiers = true}); + ABSL_CHECK_OK(status_or_runtime_builder); // Crash OK + auto runtime_builder = *std::move(status_or_runtime_builder); + ABSL_CHECK_OK(cel::EnableReferenceResolver( + runtime_builder, + cel::ReferenceResolverEnabled::kCheckedExpressionOnly)); + ABSL_CHECK_OK(cel::extensions::EnableConstantFolding( + runtime_builder, cel::internal::GetTestingMessageFactory())); + ABSL_CHECK_OK(cel::extensions::EnableOptionalTypes(runtime_builder)); + auto status_or_runtime = std::move(runtime_builder).Build(); + ABSL_CHECK_OK(status_or_runtime); // Crash OK + runtime_ = *std::move(status_or_runtime); + } + + std::unique_ptr CreateProgram(absl::string_view text) { + auto status_or_source = parser_->PrepareSource(text); + ABSL_CHECK_OK(status_or_source); // Crash OK + auto source = *std::move(status_or_source); + auto status_or_ast = parser_->Parse(*source); + ABSL_CHECK_OK(status_or_ast); // Crash OK + auto ast = *std::move(status_or_ast); + auto status_or_program = runtime_->CreateProgram(std::move(ast)); + ABSL_CHECK_OK(status_or_program); // Crash OK + return *std::move(status_or_program); + } + + void TearDown() override { + parser_.reset(); + runtime_.reset(); + } + + template + void BindProtoToActivation(absl::string_view variable, + absl::string_view text) { + T message; + ABSL_CHECK( + google::protobuf::TextFormat::ParseFromString(text, &message)); // Crash OK + activation_.InsertOrAssignValue( + variable, + cel::Value::FromMessage( + std::move(message), cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena_)); + } + + void BindJsonToActivation(absl::string_view variable, + absl::string_view text) { + google::protobuf::Value message; + ABSL_CHECK_OK( + google::protobuf::json::JsonStringToMessage(text, &message)); // Crash OK + activation_.InsertOrAssignValue( + variable, + cel::Value::FromMessage( + std::move(message), cel::internal::GetTestingDescriptorPool(), + cel::internal::GetTestingMessageFactory(), &arena_)); + } + + absl::StatusOr Evaluate(const cel::Program& program) { + return program.Evaluate(&arena_, cel::internal::GetTestingMessageFactory(), + activation_); + } + + protected: + std::unique_ptr parser_; + std::unique_ptr runtime_; + google::protobuf::Arena arena_; + cel::Activation activation_; +}; + +TEST_F(ModernUnknownTest, And) { + auto program = CreateProgram(R"cel(var1 && var2)cel"); + + { + // unk && true + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::TrueValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // true && unk + activation_.InsertOrAssignValue("var1", cel::TrueValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // false && unk + activation_.InsertOrAssignValue("var1", cel::FalseValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsFalse()); + } + { + // unk && false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsFalse()); + } + { + // error && unk + activation_.InsertOrAssignValue( + "var1", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // unk && error + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue( + "var2", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // unk && unk + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); + } +} + +TEST_F(ModernUnknownTest, Or) { + auto program = CreateProgram(R"cel(var1 || var2)cel"); + + { + // unk || true + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::TrueValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); + } + { + // true || unk + activation_.InsertOrAssignValue("var1", cel::TrueValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsTrue()); + } + { + // false || unk + activation_.InsertOrAssignValue("var1", cel::FalseValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // unk || false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // error || unk + activation_.InsertOrAssignValue( + "var1", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // unk || error + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue( + "var2", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // unk || unk + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); + } +} + +TEST_F(ModernUnknownTest, Ternary) { + auto program = CreateProgram(R"cel(var1 ? var2 : var3)cel"); + + { + // unk ? true : false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::TrueValue()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // true ? unk : false + activation_.InsertOrAssignValue("var1", cel::TrueValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var2"))); + } + { + // false ? unk : false + activation_.InsertOrAssignValue("var1", cel::FalseValue()); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsFalse()); + } + { + // error ? unk : false + activation_.InsertOrAssignValue( + "var1", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var2")}), + IsOk()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsError()); + } + { + // unk ? error : false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + activation_.InsertOrAssignValue( + "var2", cel::ErrorValue::From(absl::CancelledError(), &arena_)); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // unk ? unk : false + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + activation_.InsertOrAssignValue("var3", cel::FalseValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } + { + // unk ? true : unk + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var3")}), + IsOk()); + activation_.InsertOrAssignValue("var2", cel::TrueValue()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); + } +} + +TEST_F(ModernUnknownTest, ListUnknown) { + auto program = CreateProgram(R"cel([var1, var2])cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); +} + +TEST_F(ModernUnknownTest, ListErrorUnknown) { + auto program = CreateProgram(R"cel([var1, 1 / 0])cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(ModernUnknownTest, MapUnknown) { + auto program = CreateProgram(R"cel({"foo": var1, "bar": var2})cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); +} + +TEST_F(ModernUnknownTest, MapErrorUnknown) { + auto program = CreateProgram(R"cel({"foo": var1, "bar": 1 / 0})cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(ModernUnknownTest, StructUnknown) { + auto program = CreateProgram( + R"cel(cel.expr.conformance.proto3.TestAllTypes{single_int64: var1, single_uint64: var2})cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1"), + cel::AttributePattern("var2")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT( + result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"), cel::Attribute("var2"))); +} + +TEST_F(ModernUnknownTest, StructErrorUnknown) { + auto program = CreateProgram( + R"cel(cel.expr.conformance.proto3.TestAllTypes{single_int64: var1, single_uint64: 1u / 0u})cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(ModernUnknownTest, Variable) { + auto program = CreateProgram(R"cel(var1)cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern("var1")}), + IsOk()); + BindJsonToActivation("var1", R"json({})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute("var1"))); +} + +TEST_F(ModernUnknownTest, ReturnPartiallyUnknownValue) { + // This is an unfortunate but existing bug that we should eventually fix. We + // return a partial unknown from evaluation. + auto program = CreateProgram(R"cel(var1)cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("var1", R"json({})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsMap()); +} + +TEST_F(ModernUnknownTest, MapVariableSelectStringQualifier) { + auto program = CreateProgram(R"cel(var1.k)cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::OfString("k")})}), + IsOk()); + BindJsonToActivation("var1", R"json({})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute( + "var1", {cel::AttributeQualifier::OfString("k")}))); +} + +TEST_F(ModernUnknownTest, MapVariableSelectWildcardQualifier) { + auto program = CreateProgram(R"cel(var1.k)cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("var1", R"json({})json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre(cel::Attribute( + "var1", {cel::AttributeQualifier::OfString("k")}))); +} + +TEST_F(ModernUnknownTest, ListVariableIndexIntQualifier) { + auto program = CreateProgram(R"cel(var1[0])cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::OfInt(0)})}), + IsOk()); + BindJsonToActivation("var1", R"json([])json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre( + cel::Attribute("var1", {cel::AttributeQualifier::OfInt(0)}))); +} + +TEST_F(ModernUnknownTest, ListVariableIndexWildcardQualifier) { + auto program = CreateProgram(R"cel(var1[0])cel"); + ASSERT_THAT(activation_.SetUnknownPatterns({cel::AttributePattern( + "var1", {cel::AttributeQualifierPattern::Wildcard()})}), + IsOk()); + BindJsonToActivation("var1", R"json([])json"); + ASSERT_OK_AND_ASSIGN(auto result, Evaluate(*program)); + ASSERT_TRUE(result.IsUnknown()); + EXPECT_THAT(result.GetUnknown().ToAttributeSet(), + UnorderedElementsAre( + cel::Attribute("var1", {cel::AttributeQualifier::OfInt(0)}))); +} + +} // namespace From b5e6a2fa6be5053a52b7f383c8a749128ff50bac Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Mon, 12 Oct 2026 12:23:38 -0700 Subject: [PATCH 289/350] Add generated message fast-paths for JSON well-known types. `ParsedJsonMapValue`, `ParsedJsonListValue`, and `ParsedJsonValue` previously always accessed `google.protobuf.Struct`, `google.protobuf.ListValue`, and `google.protobuf.Value` via descriptor/reflection lookups (`FindFieldByNumber`, `GetReflection()`, `MapSize`, `LookupMapValue`, `GetRepeatedMessage`, `ListFields`), and `ParsedJsonValue` copied `string_value` onto the evaluation arena even when the `google.protobuf.Value` message was already arena-allocated. Try `proto2::DynamicCastMessage` to the generated `google::protobuf::Struct`, `google::protobuf::ListValue`, and `google::protobuf::Value` types first (falling back to reflection for dynamic messages), and use `StringValue::Wrap` when `value->GetArena() != nullptr` so arena-owned `string_value` strings are borrowed zero-copy. PiperOrigin-RevId: 998080226 --- common/values/parsed_json_list_value.cc | 86 +++++++++++++++++++++---- common/values/parsed_json_map_value.cc | 25 +++++++ common/values/parsed_json_value.cc | 43 +++++++++++++ 3 files changed, 143 insertions(+), 11 deletions(-) diff --git a/common/values/parsed_json_list_value.cc b/common/values/parsed_json_list_value.cc index 9418aebad..902deab33 100644 --- a/common/values/parsed_json_list_value.cc +++ b/common/values/parsed_json_list_value.cc @@ -41,6 +41,7 @@ #include "google/protobuf/descriptor.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/message.h" +#include "google/protobuf/message_lite.h" namespace cel { @@ -204,6 +205,12 @@ size_t ParsedJsonListValue::Size() const { if (value_ == nullptr) { return 0; } + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + return static_cast( + well_known_types::ListValueReflection::ValuesSize(*generated_message)); + } return static_cast( well_known_types::GetListValueReflectionOrDie(value_->GetDescriptor()) .ValuesSize(*value_)); @@ -223,6 +230,21 @@ absl::Status ParsedJsonListValue::Get( *result = IndexOutOfBoundsError(index, arena); return absl::OkStatus(); } + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + const int size = + well_known_types::ListValueReflection::ValuesSize(*generated_message); + if (ABSL_PREDICT_FALSE(index >= static_cast(size))) { + *result = IndexOutOfBoundsError(index, arena); + return absl::OkStatus(); + } + *result = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_message, + static_cast(index)), + arena); + return absl::OkStatus(); + } const auto reflection = well_known_types::GetListValueReflectionOrDie(value_->GetDescriptor()); if (ABSL_PREDICT_FALSE(index >= @@ -248,6 +270,22 @@ absl::Status ParsedJsonListValue::ForEach( return absl::OkStatus(); } Value scratch; + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + const int size = + well_known_types::ListValueReflection::ValuesSize(*generated_message); + for (int i = 0; i < size; ++i) { + scratch = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_message, i), + arena); + CEL_ASSIGN_OR_RETURN(auto ok, callback(static_cast(i), scratch)); + if (!ok) { + break; + } + } + return absl::OkStatus(); + } const auto reflection = well_known_types::GetListValueReflectionOrDie(value_->GetDescriptor()); const int size = reflection.ValuesSize(*value_); @@ -269,9 +307,16 @@ class ParsedJsonListValueIterator final : public ValueIterator { explicit ParsedJsonListValueIterator( const google::protobuf::Message* absl_nonnull message) : message_(message), - reflection_(well_known_types::GetListValueReflectionOrDie( - message_->GetDescriptor())), - size_(reflection_.ValuesSize(*message_)) {} + generated_( + google::protobuf::DynamicCastMessage(message_)) { + if (ABSL_PREDICT_TRUE(generated_ != nullptr)) { + size_ = well_known_types::ListValueReflection::ValuesSize(*generated_); + } else { + reflection_ = well_known_types::GetListValueReflectionOrDie( + message_->GetDescriptor()); + size_ = reflection_.ValuesSize(*message_); + } + } bool HasNext() override { return index_ < size_; } @@ -289,8 +334,14 @@ class ParsedJsonListValueIterator final : public ValueIterator { "`ValueIterator::Next` called after `ValueIterator::HasNext` " "returned false"); } - *result = common_internal::ParsedJsonValue( - &reflection_.Values(*message_, index_), arena); + if (ABSL_PREDICT_TRUE(generated_ != nullptr)) { + *result = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_, index_), + arena); + } else { + *result = common_internal::ParsedJsonValue( + &reflection_.Values(*message_, index_), arena); + } ++index_; return absl::OkStatus(); } @@ -308,8 +359,14 @@ class ParsedJsonListValueIterator final : public ValueIterator { if (index_ >= size_) { return false; } - *key_or_value = common_internal::ParsedJsonValue( - &reflection_.Values(*message_, index_), arena); + if (ABSL_PREDICT_TRUE(generated_ != nullptr)) { + *key_or_value = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_, index_), + arena); + } else { + *key_or_value = common_internal::ParsedJsonValue( + &reflection_.Values(*message_, index_), arena); + } ++index_; return true; } @@ -328,8 +385,14 @@ class ParsedJsonListValueIterator final : public ValueIterator { return false; } if (value != nullptr) { - *value = common_internal::ParsedJsonValue( - &reflection_.Values(*message_, index_), arena); + if (ABSL_PREDICT_TRUE(generated_ != nullptr)) { + *value = common_internal::ParsedJsonValue( + &well_known_types::ListValueReflection::Values(*generated_, index_), + arena); + } else { + *value = common_internal::ParsedJsonValue( + &reflection_.Values(*message_, index_), arena); + } } *key = IntValue(index_); ++index_; @@ -338,8 +401,9 @@ class ParsedJsonListValueIterator final : public ValueIterator { private: const google::protobuf::Message* absl_nonnull const message_; - const well_known_types::ListValueReflection reflection_; - const int size_; + const google::protobuf::ListValue* absl_nullable const generated_; + well_known_types::ListValueReflection reflection_; + int size_; int index_ = 0; }; diff --git a/common/values/parsed_json_map_value.cc b/common/values/parsed_json_map_value.cc index 1b568e046..c1d9ed6cb 100644 --- a/common/values/parsed_json_map_value.cc +++ b/common/values/parsed_json_map_value.cc @@ -203,6 +203,12 @@ size_t ParsedJsonMapValue::Size() const { if (value_ == nullptr) { return 0; } + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + return static_cast( + well_known_types::StructReflection::FieldsSize(*generated_message)); + } return static_cast( well_known_types::GetStructReflectionOrDie(value_->GetDescriptor()) .FieldsSize(*value_)); @@ -237,6 +243,18 @@ absl::StatusOr ParsedJsonMapValue::Find( return false; } std::string key_scratch; + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + if (const auto* value = well_known_types::StructReflection::FindField( + *generated_message, string_key->ToStringView(&key_scratch)); + value != nullptr) { + *result = common_internal::ParsedJsonValue(value, arena); + return true; + } + *result = NullValue(); + return false; + } if (const auto* value = well_known_types::GetStructReflectionOrDie( value_->GetDescriptor()) @@ -269,6 +287,13 @@ absl::Status ParsedJsonMapValue::Has( return absl::OkStatus(); } std::string key_scratch; + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(value_); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + *result = BoolValue(well_known_types::StructReflection::ContainsField( + *generated_message, string_key->ToStringView(&key_scratch))); + return absl::OkStatus(); + } if (const auto* value = well_known_types::GetStructReflectionOrDie( value_->GetDescriptor()) diff --git a/common/values/parsed_json_value.cc b/common/values/parsed_json_value.cc index fdcd5a1d0..44bf872d1 100644 --- a/common/values/parsed_json_value.cc +++ b/common/values/parsed_json_value.cc @@ -17,8 +17,10 @@ #include #include +#include "google/protobuf/struct.pb.h" #include "absl/base/attributes.h" #include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/functional/overload.h" #include "absl/status/status.h" #include "absl/strings/cord.h" @@ -29,6 +31,7 @@ #include "internal/well_known_types.h" #include "google/protobuf/arena.h" #include "google/protobuf/message.h" +#include "google/protobuf/message_lite.h" namespace cel::common_internal { @@ -36,6 +39,7 @@ namespace { using ::cel::well_known_types::AsVariant; using ::cel::well_known_types::GetValueReflectionOrDie; +using ::cel::well_known_types::ValueReflection; google::protobuf::Arena* absl_nonnull MessageArenaOr( const google::protobuf::Message* absl_nonnull message, @@ -51,6 +55,45 @@ google::protobuf::Arena* absl_nonnull MessageArenaOr( Value ParsedJsonValue(const google::protobuf::Message* absl_nonnull message, google::protobuf::Arena* absl_nonnull arena) { + if (const auto* generated_message = + google::protobuf::DynamicCastMessage(message); + ABSL_PREDICT_TRUE(generated_message != nullptr)) { + const auto kind_case = ValueReflection::GetKindCase(*generated_message); + switch (kind_case) { + case google::protobuf::Value::KIND_NOT_SET: + ABSL_FALLTHROUGH_INTENDED; + case google::protobuf::Value::kNullValue: + return NullValue(); + case google::protobuf::Value::kBoolValue: + return BoolValue(ValueReflection::GetBoolValue(*generated_message)); + case google::protobuf::Value::kNumberValue: + return DoubleValue(ValueReflection::GetNumberValue(*generated_message)); + case google::protobuf::Value::kStringValue: { + absl::string_view string = + ValueReflection::GetStringValue(*generated_message); + if (string.empty()) { + return StringValue(); + } + if (google::protobuf::Arena* message_arena = generated_message->GetArena(); + message_arena != nullptr) { + return StringValue::Wrap(string, message_arena); + } + return StringValue::From(string, arena); + } + case google::protobuf::Value::kListValue: + return ParsedJsonListValue( + &ValueReflection::GetListValue(*generated_message), + MessageArenaOr(generated_message, arena)); + case google::protobuf::Value::kStructValue: + return ParsedJsonMapValue( + &ValueReflection::GetStructValue(*generated_message), + MessageArenaOr(generated_message, arena)); + default: + return ErrorValue::From(absl::InvalidArgumentError(absl::StrCat( + "unexpected value kind case: ", kind_case)), + arena); + } + } const auto reflection = GetValueReflectionOrDie(message->GetDescriptor()); const auto kind_case = reflection.GetKindCase(*message); switch (kind_case) { From 9a97135f1954708d6d073b44a7e519c48fe16e0c Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 12 Oct 2026 12:50:27 -0700 Subject: [PATCH 290/350] Add `GetOwningArena()` to compat list and map values This is necessary for optimizations related to interop between the legacy API and modern API. PiperOrigin-RevId: 998096115 --- common/values/custom_list_value.cc | 2 ++ common/values/custom_map_value.cc | 2 ++ common/values/list_value_builder.h | 6 ++++++ common/values/map_value_builder.h | 6 ++++++ common/values/value_builder.cc | 16 ++++++++++++++++ 5 files changed, 32 insertions(+) diff --git a/common/values/custom_list_value.cc b/common/values/custom_list_value.cc index 0e55115b0..5a34b42cf 100644 --- a/common/values/custom_list_value.cc +++ b/common/values/custom_list_value.cc @@ -53,6 +53,8 @@ class EmptyListValue final : public common_internal::CompatListValue { EmptyListValue() = default; + google::protobuf::Arena* GetOwningArena() const override { return nullptr; } + std::string DebugString() const override { return "[]"; } bool IsEmpty() const override { return true; } diff --git a/common/values/custom_map_value.cc b/common/values/custom_map_value.cc index 99e610efe..ce463f890 100644 --- a/common/values/custom_map_value.cc +++ b/common/values/custom_map_value.cc @@ -70,6 +70,8 @@ class EmptyMapValue final : public common_internal::CompatMapValue { EmptyMapValue() = default; + google::protobuf::Arena* GetOwningArena() const override { return nullptr; } + std::string DebugString() const override { return "{}"; } bool IsEmpty() const override { return true; } diff --git a/common/values/list_value_builder.h b/common/values/list_value_builder.h index 91cef066d..d7af362c9 100644 --- a/common/values/list_value_builder.h +++ b/common/values/list_value_builder.h @@ -39,6 +39,9 @@ namespace common_internal { // `list_value_builder.cc`. class CompatListValue : public CustomListValueInterface, public google::api::expr::runtime::CelList { + public: + virtual google::protobuf::Arena* GetOwningArena() const = 0; + private: NativeTypeId GetNativeTypeId() const final { return NativeTypeId::For(); @@ -78,6 +81,9 @@ class MutableListValue : public CustomListValueInterface { // inheritance and `dynamic_cast`. class MutableCompatListValue : public MutableListValue, public google::api::expr::runtime::CelList { + public: + virtual google::protobuf::Arena* GetOwningArena() const = 0; + private: NativeTypeId GetNativeTypeId() const final { return NativeTypeId::For(); diff --git a/common/values/map_value_builder.h b/common/values/map_value_builder.h index a5a47eda9..2eb70b613 100644 --- a/common/values/map_value_builder.h +++ b/common/values/map_value_builder.h @@ -39,6 +39,9 @@ namespace common_internal { // `map_value_builder.cc`. class CompatMapValue : public CustomMapValueInterface, public google::api::expr::runtime::CelMap { + public: + virtual google::protobuf::Arena* GetOwningArena() const = 0; + private: NativeTypeId GetNativeTypeId() const final { return NativeTypeId::For(); @@ -78,6 +81,9 @@ class MutableMapValue : public CustomMapValueInterface { // inheritance and `dynamic_cast`. class MutableCompatMapValue : public MutableMapValue, public google::api::expr::runtime::CelMap { + public: + virtual google::protobuf::Arena* GetOwningArena() const = 0; + private: NativeTypeId GetNativeTypeId() const final { return NativeTypeId::For(); diff --git a/common/values/value_builder.cc b/common/values/value_builder.cc index 78d7a57d8..0bcd29eb8 100644 --- a/common/values/value_builder.cc +++ b/common/values/value_builder.cc @@ -251,6 +251,10 @@ class CompatListValueImpl final : public CompatListValue { explicit CompatListValueImpl(ValueVector&& elements) : elements_(std::move(elements)) {} + google::protobuf::Arena* GetOwningArena() const override { + return elements_.get_allocator().arena(); + } + std::string DebugString() const override { return absl::StrCat("[", absl::StrJoin(elements_, ", ", ValueFormatter{}), "]"); @@ -388,6 +392,10 @@ class MutableCompatListValueImpl final : public MutableCompatListValue { explicit MutableCompatListValueImpl(google::protobuf::Arena* absl_nonnull arena) : elements_(arena) {} + google::protobuf::Arena* GetOwningArena() const override { + return elements_.get_allocator().arena(); + } + std::string DebugString() const override { return absl::StrCat("[", absl::StrJoin(elements_, ", ", ValueFormatter{}), "]"); @@ -995,6 +1003,10 @@ class CompatMapValueImpl final : public CompatMapValue { public: explicit CompatMapValueImpl(ValueFlatHashMap&& map) : map_(std::move(map)) {} + google::protobuf::Arena* GetOwningArena() const override { + return map_.get_allocator().arena(); + } + std::string DebugString() const override { return absl::StrCat("{", absl::StrJoin(map_, ", ", ValueFormatter{}), "}"); } @@ -1159,6 +1171,10 @@ class TrivialMutableMapValueImpl final : public MutableCompatMapValue { explicit TrivialMutableMapValueImpl(google::protobuf::Arena* absl_nonnull arena) : map_(arena) {} + google::protobuf::Arena* GetOwningArena() const override { + return map_.get_allocator().arena(); + } + std::string DebugString() const override { return absl::StrCat("{", absl::StrJoin(map_, ", ", ValueFormatter{}), "}"); } From cffec7c53b2404d0407c8d5b09471ecdd86cc65c Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 12 Oct 2026 14:27:29 -0700 Subject: [PATCH 291/350] Make `cel::Value` trivial again PiperOrigin-RevId: 998157223 --- common/BUILD | 1 + common/legacy_value.cc | 7 +- common/value.h | 49 --- common/values/unknown_value.cc | 11 + common/values/unknown_value.h | 25 +- common/values/value_variant.cc | 209 ----------- common/values/value_variant.h | 484 ++++---------------------- common/values/value_variant_test.cc | 1 + eval/compiler/instrumentation_test.cc | 2 + eval/eval/BUILD | 1 + eval/eval/attribute_utility.cc | 34 +- eval/eval/attribute_utility.h | 14 +- eval/eval/attribute_utility_test.cc | 10 +- eval/eval/comprehension_step.cc | 7 +- eval/eval/container_access_step.cc | 5 +- eval/eval/create_list_step.cc | 2 +- eval/eval/create_map_step.cc | 3 +- eval/eval/create_struct_step.cc | 2 +- eval/eval/equality_steps.cc | 4 +- eval/eval/evaluator_core.cc | 2 +- eval/eval/function_step.cc | 10 +- eval/eval/ident_step.cc | 4 +- eval/eval/logic_step.cc | 4 +- eval/eval/select_step.cc | 3 +- extensions/select_optimization.cc | 2 +- 25 files changed, 172 insertions(+), 724 deletions(-) delete mode 100644 common/values/value_variant.cc diff --git a/common/BUILD b/common/BUILD index 27ae6ded9..213a91035 100644 --- a/common/BUILD +++ b/common/BUILD @@ -756,6 +756,7 @@ cc_library( ":unknown", ":value_kind", "//base:attributes", + "//base:function_result_set", "//common/internal:byte_string", "//eval/internal:cel_value_equal", "//eval/public:cel_value", diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 505f2fbf5..622257f43 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -1071,8 +1071,8 @@ absl::Status ModernValue(google::protobuf::Arena* arena, MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); return absl::OkStatus(); case CelValue::Type::kUnknownSet: - result = - common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie()); + result = common_internal::MakeUnknownValue( + *legacy_value.UnknownSetOrDie(), arena); return absl::OkStatus(); case CelValue::Type::kCelType: { auto type_name = legacy_value.CelTypeOrDie().value(); @@ -1183,7 +1183,8 @@ absl::StatusOr FromLegacyValue(google::protobuf::Arena* arena, case CelValue::Type::kMap: return MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); case CelValue::Type::kUnknownSet: - return common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie()); + return common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie(), + arena); case CelValue::Type::kCelType: return CreateTypeValueFromView(arena, legacy_value.CelTypeOrDie().value()); diff --git a/common/value.h b/common/value.h index 9278cf4de..b8801c00e 100644 --- a/common/value.h +++ b/common/value.h @@ -298,103 +298,54 @@ class Value final : private common_internal::ValueMixin { Value() = default; Value(const Value&) = default; Value& operator=(const Value&) = default; - Value(Value&& other) = default; - Value& operator=(Value&&) = default; // NOLINTNEXTLINE(google-explicit-constructor) Value(const ListValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(ListValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const ListValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(ListValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const MapValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(MapValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const MapValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(MapValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const StructValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(StructValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const StructValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(StructValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const MessageValue& value) : variant_(value.ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(MessageValue&& value) : variant_(std::move(value).ToValueVariant()) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const MessageValue& value) { variant_ = value.ToValueVariant(); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(MessageValue&& value) { - variant_ = std::move(value).ToValueVariant(); - return *this; - } - // NOLINTNEXTLINE(google-explicit-constructor) Value(const OptionalValue& value) : variant_(absl::in_place_type, static_cast(value)) {} - // NOLINTNEXTLINE(google-explicit-constructor) - Value(OptionalValue&& value) - : variant_(absl::in_place_type, - static_cast(value)) {} - // NOLINTNEXTLINE(google-explicit-constructor) Value& operator=(const OptionalValue& value) { variant_.Assign(static_cast(value)); return *this; } - // NOLINTNEXTLINE(google-explicit-constructor) - Value& operator=(OptionalValue&& value) { - variant_.Assign(static_cast(value)); - return *this; - } - template >>> diff --git a/common/values/unknown_value.cc b/common/values/unknown_value.cc index 1cb8a7674..abfd66625 100644 --- a/common/values/unknown_value.cc +++ b/common/values/unknown_value.cc @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/base/no_destructor.h" #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include "base/function_result_set.h" #include "common/value.h" #include "google/protobuf/arena.h" #include "google/protobuf/descriptor.h" @@ -63,4 +65,13 @@ absl::Status UnknownValue::Equal( return absl::OkStatus(); } +namespace common_internal { + +const FunctionResultSet& GetEmptyFunctionResults() { + static const absl::NoDestructor instance; + return *instance; +} + +} // namespace common_internal + } // namespace cel diff --git a/common/values/unknown_value.h b/common/values/unknown_value.h index 1d17de1ca..dae2aa0b2 100644 --- a/common/values/unknown_value.h +++ b/common/values/unknown_value.h @@ -26,6 +26,7 @@ #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" +#include "base/function_result_set.h" #include "common/type.h" #include "common/unknown.h" #include "common/value_kind.h" @@ -42,7 +43,7 @@ class UnknownValue; namespace common_internal { [[nodiscard]] -UnknownValue MakeUnknownValue(Unknown value); +UnknownValue MakeUnknownValue(Unknown value, google::protobuf::Arena* absl_nonnull arena); [[nodiscard]] Unknown GetUnknown(const UnknownValue& value); [[nodiscard]] @@ -90,7 +91,8 @@ class UnknownValue final : private common_internal::ValueMixin { [[nodiscard]] AttributeSet ToAttributeSet() const { - return unknown_.unknown_attributes(); + return unknown_ != nullptr ? unknown_->unknown_attributes() + : AttributeSet(); } void swap(UnknownValue& other) noexcept { @@ -99,15 +101,16 @@ class UnknownValue final : private common_internal::ValueMixin { } private: - friend UnknownValue common_internal::MakeUnknownValue(Unknown value); + friend UnknownValue common_internal::MakeUnknownValue( + Unknown value, google::protobuf::Arena* absl_nonnull arena); friend Unknown common_internal::GetUnknown(const UnknownValue&); friend const FunctionResultSet& common_internal::GetUnknownFunctionResultSet( const UnknownValue& value); friend class common_internal::ValueMixin; - explicit UnknownValue(Unknown unknown) : unknown_(std::move(unknown)) {} + explicit UnknownValue(const Unknown* unknown) : unknown_(unknown) {} - Unknown unknown_; + const Unknown* unknown_ = nullptr; }; inline void swap(UnknownValue& lhs, UnknownValue& rhs) noexcept { @@ -120,20 +123,24 @@ inline std::ostream& operator<<(std::ostream& out, const UnknownValue& value) { namespace common_internal { +const FunctionResultSet& GetEmptyFunctionResults(); + [[nodiscard]] -inline UnknownValue MakeUnknownValue(Unknown value) { - return UnknownValue(std::move(value)); +inline UnknownValue MakeUnknownValue(Unknown value, + google::protobuf::Arena* absl_nonnull arena) { + return UnknownValue(google::protobuf::Arena::Create(arena, std::move(value))); } [[nodiscard]] inline Unknown GetUnknown(const UnknownValue& value) { - return value.unknown_; + return value.unknown_ != nullptr ? *value.unknown_ : Unknown(); } [[nodiscard]] inline const FunctionResultSet& GetUnknownFunctionResultSet( const UnknownValue& value ABSL_ATTRIBUTE_LIFETIME_BOUND) { - return value.unknown_.unknown_function_results(); + return value.unknown_ != nullptr ? value.unknown_->unknown_function_results() + : GetEmptyFunctionResults(); } } // namespace common_internal diff --git a/common/values/value_variant.cc b/common/values/value_variant.cc deleted file mode 100644 index 5f240f79b..000000000 --- a/common/values/value_variant.cc +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/values/value_variant.h" - -#include -#include -#include -#include - -#include "absl/base/optimization.h" -#include "absl/log/absl_check.h" -#include "common/values/unknown_value.h" -#include "common/values/values.h" - -namespace cel::common_internal { - -void ValueVariant::SlowCopyConstruct(const ValueVariant& other) noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowMoveConstruct(ValueVariant& other) noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowDestruct() noexcept { - ABSL_DCHECK((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial); - - switch (index_) { - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } -} - -void ValueVariant::SlowCopyAssign(const ValueVariant& other, bool trivial, - bool other_trivial) noexcept { - ABSL_DCHECK(!trivial || !other_trivial); - - if (trivial) { - switch (other.index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - } else if (other_trivial) { - switch (index_) { - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - FastCopyAssign(other); - } else { - switch (index_) { - case ValueIndex::kUnknown: - switch (other.index_) { - case ValueIndex::kUnknown: - At()->~UnknownValue(); - ::new (static_cast(&raw_[0])) - UnknownValue(*other.At()); - index_ = other.index_; - kind_ = other.kind_; - break; - default: - ABSL_UNREACHABLE(); - } - break; - default: - ABSL_UNREACHABLE(); - } - flags_ = other.flags_; - } -} - -void ValueVariant::SlowMoveAssign(ValueVariant& other, bool trivial, - bool other_trivial) noexcept { - ABSL_DCHECK(!trivial || !other_trivial); - - if (trivial) { - switch (other.index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&raw_[0])) - UnknownValue(std::move(*other.At())); - break; - default: - ABSL_UNREACHABLE(); - } - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - } else if (other_trivial) { - switch (index_) { - case ValueIndex::kUnknown: - At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - FastMoveAssign(other); - } else { - switch (index_) { - case ValueIndex::kUnknown: - switch (other.index_) { - case ValueIndex::kUnknown: - *At() = std::move(*other.At()); - break; - default: - ABSL_UNREACHABLE(); - } - break; - default: - ABSL_UNREACHABLE(); - } - flags_ = other.flags_; - } -} - -void ValueVariant::SlowSwap(ValueVariant& lhs, ValueVariant& rhs, - bool lhs_trivial, bool rhs_trivial) noexcept { - using std::swap; - ABSL_DCHECK(!lhs_trivial || !rhs_trivial); - - if (lhs_trivial) { - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(lhs), sizeof(ValueVariant)); - switch (rhs.index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&lhs.raw_[0])) - UnknownValue(*rhs.At()); - rhs.At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - lhs.index_ = rhs.index_; - lhs.kind_ = rhs.kind_; - lhs.flags_ = rhs.flags_; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(rhs), tmp, sizeof(ValueVariant)); - } else if (rhs_trivial) { - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(rhs), sizeof(ValueVariant)); - switch (lhs.index_) { - case ValueIndex::kUnknown: - ::new (static_cast(&rhs.raw_[0])) - UnknownValue(*lhs.At()); - lhs.At()->~UnknownValue(); - break; - default: - ABSL_UNREACHABLE(); - } - rhs.index_ = lhs.index_; - rhs.kind_ = lhs.kind_; - rhs.flags_ = lhs.flags_; - // This is acceptable. We know that both are trivially copyable at runtime. - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(lhs), tmp, sizeof(ValueVariant)); - } else { - ValueVariant tmp = std::move(lhs); - lhs = std::move(rhs); - rhs = std::move(tmp); - } -} - -} // namespace cel::common_internal diff --git a/common/values/value_variant.h b/common/values/value_variant.h index e7c199834..aaf0766d2 100644 --- a/common/values/value_variant.h +++ b/common/values/value_variant.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -27,8 +26,6 @@ #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/meta/type_traits.h" -#include "absl/utility/utility.h" -#include "common/arena.h" #include "common/value_kind.h" #include "common/values/bool_value.h" #include "common/values/bytes_value.h" @@ -89,23 +86,9 @@ enum class ValueIndex : uint8_t { kBytes, kString, kError, - // Keep non-trivial alternatives together to aid in compiling optimizations. kUnknown, }; -// Used by ValueVariant to indicate pre-computed behaviors. -enum class ValueFlags : uint32_t { - kNone = 0, - kNonTrivial = 1, -}; - -ABSL_ATTRIBUTE_ALWAYS_INLINE inline constexpr ValueFlags operator&( - ValueFlags lhs, ValueFlags rhs) { - return static_cast( - static_cast>(lhs) & - static_cast>(rhs)); -} - // Traits specialized by each alternative. // // ValueIndex ValueAlternative::kIndex @@ -130,166 +113,90 @@ template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kNull; static constexpr ValueKind kKind = NullValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const NullValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kBool; static constexpr ValueKind kKind = BoolValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const BoolValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kInt; static constexpr ValueKind kKind = IntValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const IntValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kUint; static constexpr ValueKind kKind = UintValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const UintValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kDouble; static constexpr ValueKind kKind = DoubleValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const DoubleValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kDuration; static constexpr ValueKind kKind = DurationValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const DurationValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kTimestamp; static constexpr ValueKind kKind = TimestampValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const TimestampValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kType; static constexpr ValueKind kKind = TypeValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const TypeValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyList; static constexpr ValueKind kKind = LegacyListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedJsonList; static constexpr ValueKind kKind = ParsedJsonListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedJsonListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedRepeatedField; static constexpr ValueKind kKind = ParsedRepeatedFieldValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags( - const ParsedRepeatedFieldValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomList; static constexpr ValueKind kKind = CustomListValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomListValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyMap; static constexpr ValueKind kKind = LegacyMapValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedJsonMap; static constexpr ValueKind kKind = ParsedJsonMapValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedJsonMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedMapField; static constexpr ValueKind kKind = ParsedMapFieldValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedMapFieldValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> @@ -297,98 +204,54 @@ struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomMap; static constexpr ValueKind kKind = CustomMapValue::kKind; static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomMapValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kLegacyStruct; static constexpr ValueKind kKind = LegacyStructValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const LegacyStructValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kParsedMessage; static constexpr ValueKind kKind = ParsedMessageValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const ParsedMessageValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kCustomStruct; static constexpr ValueKind kKind = CustomStructValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const CustomStructValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kOpaque; static constexpr ValueKind kKind = OpaqueValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static constexpr ValueFlags Flags(const OpaqueValue* absl_nonnull) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kBytes; static constexpr ValueKind kKind = BytesValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static ValueFlags Flags(const BytesValue* absl_nonnull alternative) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kString; static constexpr ValueKind kKind = StringValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static ValueFlags Flags(const StringValue* absl_nonnull alternative) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kError; static constexpr ValueKind kKind = ErrorValue::kKind; - static constexpr bool kAlwaysTrivial = true; - - static ValueFlags Flags(const ErrorValue* absl_nonnull alternative) { - return ValueFlags::kNone; - } }; template <> struct ValueAlternative { static constexpr ValueIndex kIndex = ValueIndex::kUnknown; static constexpr ValueKind kKind = UnknownValue::kKind; - static constexpr bool kAlwaysTrivial = false; - - static constexpr ValueFlags Flags(const UnknownValue* absl_nonnull) { - return ValueFlags::kNonTrivial; - } }; template @@ -406,93 +269,33 @@ inline constexpr bool IsValueAlternativeV = IsValueAlternative::value; inline constexpr size_t kValueVariantAlign = 8; inline constexpr size_t kValueVariantSize = 24; -// Hand-rolled variant used by cel::Value which exhibits up to a 25% performance -// improvement compared to using std::variant. -// -// The implementation abuses the fact that most alternatives are trivially -// copyable and some are conditionally trivially copyable at runtime. For the -// fast path, we perform raw byte copying. For the slow path, we fallback to a -// non-inlined function. The compiler is typically smart enough to inline the -// fast path and emit efficient instructions for the raw byte copying (usually -// two instructions). It also uses switch for visiting, which most compilers can -// optimize better compared to a function pointer table (which libc++ currently -// uses and Clang currently does not optimize well). -class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI - ValueVariant final { +// Hand-rolled variant used by cel::Value which avoids std::variant and its +// lack of support for an unchecked get. +class alignas(kValueVariantAlign) ValueVariant { public: ValueVariant() = default; - - ValueVariant(const ValueVariant& other) noexcept - : index_(other.index_), kind_(other.kind_), flags_(other.flags_) { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone) { - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } else { - SlowCopyConstruct(other); - } - } - - ValueVariant(ValueVariant&& other) noexcept - : index_(other.index_), kind_(other.kind_), flags_(other.flags_) { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone) { - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } else { - SlowMoveConstruct(other); - } - } - - ~ValueVariant() { - if ((flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNonTrivial) { - SlowDestruct(); - } - } - - ValueVariant& operator=(const ValueVariant& other) noexcept { - if (this != &other) { - const bool trivial = - (flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool other_trivial = - (other.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (trivial && other_trivial) { - FastCopyAssign(other); - } else { - SlowCopyAssign(other, trivial, other_trivial); - } - } - return *this; - } - - ValueVariant& operator=(ValueVariant&& other) noexcept { - if (this != &other) { - const bool trivial = - (flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool other_trivial = - (other.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (trivial && other_trivial) { - FastMoveAssign(other); - } else { - SlowMoveAssign(other, trivial, other_trivial); - } - } - return *this; - } + ValueVariant(const ValueVariant&) = default; + ValueVariant& operator=(const ValueVariant&) = default; template - explicit ValueVariant(absl::in_place_type_t, Args&&... args) + explicit ValueVariant(std::in_place_type_t, Args&&... args) : index_(ValueAlternative::kIndex), kind_(ValueAlternative::kKind) { static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - T(std::forward(args)...)); + ::new (static_cast(&raw_[0])) T(std::forward(args)...); } template >>> explicit ValueVariant(T&& value) - : ValueVariant(absl::in_place_type>, + : ValueVariant(std::in_place_type>, std::forward(value)) {} - ValueKind kind() const { return kind_; } + [[nodiscard]] + ValueKind kind() const { + return kind_; + } template void Assign(T&& value) { @@ -501,67 +304,35 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI static_assert(alignof(U) <= kValueVariantAlign); static_assert(sizeof(U) <= kValueVariantSize); - if constexpr (ValueAlternative::kAlwaysTrivial) { - if ((flags_ & ValueFlags::kNonTrivial) != ValueFlags::kNone) { - SlowDestruct(); - } - index_ = ValueAlternative::kIndex; - kind_ = ValueAlternative::kKind; - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - U(std::forward(value))); - } else { - // U is not always trivial. See if the current active alternative is U. If - // it is, we can just do a simple assignment without having to destruct - // first. Otherwise fallback to destruct and construct. - if (index_ == ValueAlternative::kIndex) { - *At() = std::forward(value); - flags_ = ValueAlternative::Flags(At()); - } else { - if ((flags_ & ValueFlags::kNonTrivial) != ValueFlags::kNone) { - SlowDestruct(); - } - index_ = ValueAlternative::kIndex; - kind_ = ValueAlternative::kKind; - flags_ = ValueAlternative::Flags(::new (static_cast(&raw_[0])) - U(std::forward(value))); - } - } + index_ = ValueAlternative::kIndex; + kind_ = ValueAlternative::kKind; + ::new (static_cast(&raw_[0])) U(std::forward(value)); } template + [[nodiscard]] bool Is() const { return index_ == ValueAlternative::kIndex; } template - T& Get() & ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] + T& Get() ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(Is()); return *At(); } template - const T& Get() const& ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] + const T& Get() const ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_DCHECK(Is()); return *At(); } template - T&& Get() && ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(Is()); - - return std::move(*At()); - } - - template - const T&& Get() const&& ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_DCHECK(Is()); - - return std::move(*At()); - } - - template + [[nodiscard]] T* absl_nullable As() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (Is()) { return At(); @@ -570,6 +341,7 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template + [[nodiscard]] const T* absl_nullable As() const ABSL_ATTRIBUTE_LIFETIME_BOUND { if (Is()) { return At(); @@ -578,187 +350,119 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template - ABSL_ATTRIBUTE_ALWAYS_INLINE decltype(auto) Visit(Visitor&& visitor) & { - return std::as_const(*this).Visit(std::forward(visitor)); - } - - template - decltype(auto) Visit(Visitor&& visitor) const& { + decltype(auto) Visit(Visitor&& visitor) const { switch (index_) { case ValueIndex::kNull: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBool: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kInt: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUint: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDouble: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDuration: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kTimestamp: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kType: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedRepeatedField: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomList: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMapField: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomMap: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyStruct: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMessage: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomStruct: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kOpaque: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBytes: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kString: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kError: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUnknown: - return std::forward(visitor)(Get()); + return std::forward(visitor)(*At()); } } template - decltype(auto) Visit(Visitor&& visitor) && { + decltype(auto) Visit(Visitor&& visitor) { switch (index_) { case ValueIndex::kNull: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBool: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kInt: - return std::forward(visitor)(std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUint: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDouble: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kDuration: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kTimestamp: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kType: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedRepeatedField: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomList: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedJsonMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMapField: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomMap: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kLegacyStruct: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kParsedMessage: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kCustomStruct: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kOpaque: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kBytes: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kString: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kError: - return std::forward(visitor)( - std::move(*this).Get()); + return std::forward(visitor)(*At()); case ValueIndex::kUnknown: - return std::forward(visitor)( - std::move(*this).Get()); - } - } - - template - ABSL_ATTRIBUTE_ALWAYS_INLINE decltype(auto) Visit(Visitor&& visitor) const&& { - return Visit(std::forward(visitor)); - } - - friend void swap(ValueVariant& lhs, ValueVariant& rhs) noexcept { - if (&lhs != &rhs) { - const bool lhs_trivial = - (lhs.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - const bool rhs_trivial = - (rhs.flags_ & ValueFlags::kNonTrivial) == ValueFlags::kNone; - if (lhs_trivial && rhs_trivial) { -// We validated the instances can be copied byte-wise at runtime, but compilers -// warn since this is not safe in the general case. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" -#elif defined(__clang__) && __clang_major__ >= 20 -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wnontrivial-memcall" -#endif - alignas(ValueVariant) std::byte tmp[sizeof(ValueVariant)]; - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(tmp, std::addressof(lhs), sizeof(ValueVariant)); - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(lhs), std::addressof(rhs), - sizeof(ValueVariant)); - // NOLINTNEXTLINE(bugprone-undefined-memory-manipulation) - std::memcpy(std::addressof(rhs), tmp, sizeof(ValueVariant)); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#elif defined(__clang__) && __clang_major__ >= 20 -#pragma clang diagnostic pop -#endif - } else { - SlowSwap(lhs, rhs, lhs_trivial, rhs_trivial); - } + return std::forward(visitor)(*At()); } } private: - friend struct cel::ArenaTraits; - template + [[nodiscard]] ABSL_ATTRIBUTE_ALWAYS_INLINE T* absl_nonnull At() ABSL_ATTRIBUTE_LIFETIME_BOUND { + static_assert(IsValueAlternativeV); static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); @@ -766,59 +470,23 @@ class alignas(kValueVariantAlign) CEL_COMMON_INTERNAL_VALUE_VARIANT_TRIVIAL_ABI } template + [[nodiscard]] ABSL_ATTRIBUTE_ALWAYS_INLINE const T* absl_nonnull At() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + static_assert(IsValueAlternativeV); static_assert(alignof(T) <= kValueVariantAlign); static_assert(sizeof(T) <= kValueVariantSize); return std::launder(reinterpret_cast(&raw_[0])); } - ABSL_ATTRIBUTE_ALWAYS_INLINE void FastCopyAssign( - const ValueVariant& other) noexcept { - index_ = other.index_; - kind_ = other.kind_; - flags_ = other.flags_; - std::memcpy(raw_, other.raw_, sizeof(raw_)); - } - - ABSL_ATTRIBUTE_ALWAYS_INLINE void FastMoveAssign( - ValueVariant& other) noexcept { - FastCopyAssign(other); - } - - void SlowCopyConstruct(const ValueVariant& other) noexcept; - - void SlowMoveConstruct(ValueVariant& other) noexcept; - - void SlowDestruct() noexcept; - - void SlowCopyAssign(const ValueVariant& other, bool trivial, - bool other_trivial) noexcept; - - void SlowMoveAssign(ValueVariant& other, bool ntrivial, - bool other_trivial) noexcept; - - static void SlowSwap(ValueVariant& lhs, ValueVariant& rhs, bool lhs_trivial, - bool rhs_trivial) noexcept; - ValueIndex index_ = ValueIndex::kNull; ValueKind kind_ = ValueKind::kNull; - ValueFlags flags_ = ValueFlags::kNone; alignas(kValueVariantAlign) std::byte raw_[kValueVariantSize]; }; } // namespace common_internal -template <> -struct ArenaTraits { - static bool trivially_destructible( - const common_internal::ValueVariant& value) { - return (value.flags_ & common_internal::ValueFlags::kNonTrivial) == - common_internal::ValueFlags::kNone; - } -}; - } // namespace cel #endif // THIRD_PARTY_CEL_CPP_COMMON_VALUES_VALUE_VARIANT_H_ diff --git a/common/values/value_variant_test.cc b/common/values/value_variant_test.cc index 968dca0d9..1c0416d4c 100644 --- a/common/values/value_variant_test.cc +++ b/common/values/value_variant_test.cc @@ -110,6 +110,7 @@ TYPED_TEST(ValueVariantTest, MoveAssign) { } TYPED_TEST(ValueVariantTest, Swap) { + using std::swap; using Left = typename TypeParam::first_type; using Right = typename TypeParam::second_type; diff --git a/eval/compiler/instrumentation_test.cc b/eval/compiler/instrumentation_test.cc index 630f398d1..39edecf6a 100644 --- a/eval/compiler/instrumentation_test.cc +++ b/eval/compiler/instrumentation_test.cc @@ -105,6 +105,7 @@ TEST_F(InstrumentationTest, Basic) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); // AST for the test expression: // + <4> @@ -150,6 +151,7 @@ TEST_F(InstrumentationTest, BasicWithConstFolding) { ASSERT_OK_AND_ASSIGN(auto value, plan.EvaluateWithCallback( activation, /*embedder_context=*/nullptr, EvaluationListener(), state)); + static_cast(value); // AST for the test expression: // + <4> diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 13efe4c0e..08dacb9b2 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -819,6 +819,7 @@ cc_test( "//eval/public:unknown_set", "//internal:testing", "//runtime/internal:attribute_matcher", + "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@com_google_protobuf//:protobuf", ], diff --git a/eval/eval/attribute_utility.cc b/eval/eval/attribute_utility.cc index a0e57161a..b1e75111d 100644 --- a/eval/eval/attribute_utility.cc +++ b/eval/eval/attribute_utility.cc @@ -104,7 +104,7 @@ bool AttributeUtility::CheckForUnknown(const AttributeTrail& trail, // it together with initial_set (if initial_set is not null). // Returns pointer to merged set or nullptr, if there were no sets to merge. absl::optional AttributeUtility::MergeUnknowns( - absl::Span args) const { + absl::Span args, google::protobuf::Arena* arena) const { // Empty unknown value may be used as a sentinel in some tests so need to // distinguish unset (nullopt) and empty(engaged empty value). absl::optional result_set; @@ -129,11 +129,13 @@ absl::optional AttributeUtility::MergeUnknowns( return cel::common_internal::MakeUnknownValue( cel::Unknown(result_set->unknown_attributes(), - result_set->unknown_function_results())); + result_set->unknown_function_results()), + arena); } -UnknownValue AttributeUtility::MergeUnknownValues( - const UnknownValue& left, const UnknownValue& right) const { +UnknownValue AttributeUtility::MergeUnknownValues(const UnknownValue& left, + const UnknownValue& right, + google::protobuf::Arena* arena) const { // Empty unknown value may be used as a sentinel in some tests so need to // distinguish unset (nullopt) and empty(engaged empty value). AttributeSet attributes; @@ -145,7 +147,7 @@ UnknownValue AttributeUtility::MergeUnknownValues( cel::common_internal::GetUnknownFunctionResultSet(right)); return cel::common_internal::MakeUnknownValue( - cel::Unknown(std::move(attributes), std::move(function_results))); + cel::Unknown(std::move(attributes), std::move(function_results)), arena); } // Creates merged UnknownAttributeSet. @@ -174,7 +176,7 @@ AttributeSet AttributeUtility::CheckForUnknowns( // Returns pointer to merged set or nullptr, if there were no sets to merge. absl::optional AttributeUtility::IdentifyAndMergeUnknowns( absl::Span args, absl::Span attrs, - bool use_partial) const { + bool use_partial, google::protobuf::Arena* arena) const { absl::optional result_set; // Identify new unknowns by attribute patterns. @@ -184,7 +186,7 @@ absl::optional AttributeUtility::IdentifyAndMergeUnknowns( } // merge down existing unknown sets - absl::optional arg_unknowns = MergeUnknowns(args); + absl::optional arg_unknowns = MergeUnknowns(args, arena); if (!result_set.has_value()) { // No new unknowns so no need to check for presence of existing unknowns -- @@ -199,12 +201,14 @@ absl::optional AttributeUtility::IdentifyAndMergeUnknowns( return cel::common_internal::MakeUnknownValue( cel::Unknown(result_set->unknown_attributes(), - result_set->unknown_function_results())); + result_set->unknown_function_results()), + arena); } -UnknownValue AttributeUtility::CreateUnknownSet(cel::Attribute attr) const { +UnknownValue AttributeUtility::CreateUnknownSet(cel::Attribute attr, + google::protobuf::Arena* arena) const { return cel::common_internal::MakeUnknownValue( - cel::Unknown(AttributeSet({std::move(attr)}))); + cel::Unknown(AttributeSet({std::move(attr)})), arena); } absl::StatusOr AttributeUtility::CreateMissingAttributeError( @@ -216,9 +220,10 @@ absl::StatusOr AttributeUtility::CreateMissingAttributeError( UnknownValue AttributeUtility::CreateUnknownSet( const cel::FunctionDescriptor& fn_descriptor, int64_t expr_id, - absl::Span args) const { + absl::Span args, google::protobuf::Arena* arena) const { return cel::common_internal::MakeUnknownValue( - cel::Unknown(FunctionResultSet(FunctionResult(fn_descriptor.name())))); + cel::Unknown(FunctionResultSet(FunctionResult(fn_descriptor.name()))), + arena); } void AttributeUtility::Add(Accumulator& a, const cel::UnknownValue& v) const { @@ -257,9 +262,10 @@ bool Accumulator::IsEmpty() const { function_result_set_.empty(); } -cel::UnknownValue Accumulator::Build() && { +cel::UnknownValue Accumulator::Build(google::protobuf::Arena* arena) && { return cel::common_internal::MakeUnknownValue( - cel::Unknown(std::move(attribute_set_), std::move(function_result_set_))); + cel::Unknown(std::move(attribute_set_), std::move(function_result_set_)), + arena); } } // namespace google::api::expr::runtime diff --git a/eval/eval/attribute_utility.h b/eval/eval/attribute_utility.h index 51ffc5d47..251e6c7d2 100644 --- a/eval/eval/attribute_utility.h +++ b/eval/eval/attribute_utility.h @@ -71,7 +71,7 @@ class AttributeUtility { bool IsEmpty() const; - cel::UnknownValue Build() &&; + cel::UnknownValue Build(google::protobuf::Arena* arena) &&; private: explicit Accumulator(const AttributeUtility& parent) @@ -131,11 +131,12 @@ class AttributeUtility { // Scans over the args collection, merges any UnknownValues found. // Returns the merged UnknownValue or nullopt if not found. absl::optional MergeUnknowns( - absl::Span args) const; + absl::Span args, google::protobuf::Arena* arena) const; // Creates a merged UnknownValue from two unknown values. cel::UnknownValue MergeUnknownValues(const cel::UnknownValue& left, - const cel::UnknownValue& right) const; + const cel::UnknownValue& right, + google::protobuf::Arena* arena) const; // Creates merged UnknownValue. // Merges together UnknownValues found in the args @@ -143,10 +144,11 @@ class AttributeUtility { // Returns returns the merged UnknownValue if available or nullopt. absl::optional IdentifyAndMergeUnknowns( absl::Span args, absl::Span attrs, - bool use_partial) const; + bool use_partial, google::protobuf::Arena* arena) const; // Create an initial UnknownSet from a single attribute. - cel::UnknownValue CreateUnknownSet(cel::Attribute attr) const; + cel::UnknownValue CreateUnknownSet(cel::Attribute attr, + google::protobuf::Arena* arena) const; // Factory function for missing attribute errors. absl::StatusOr CreateMissingAttributeError( @@ -155,7 +157,7 @@ class AttributeUtility { // Create an initial UnknownSet from a single missing function call. cel::UnknownValue CreateUnknownSet( const cel::FunctionDescriptor& fn_descriptor, int64_t expr_id, - absl::Span args) const; + absl::Span args, google::protobuf::Arena* arena) const; Accumulator CreateAccumulator() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return Accumulator(*this); diff --git a/eval/eval/attribute_utility_test.cc b/eval/eval/attribute_utility_test.cc index ea42c5b3b..cf65bd8c7 100644 --- a/eval/eval/attribute_utility_test.cc +++ b/eval/eval/attribute_utility_test.cc @@ -3,6 +3,7 @@ #include #include +#include "absl/types/optional.h" #include "absl/types/span.h" #include "base/attribute.h" #include "base/attribute_set.h" @@ -86,9 +87,9 @@ TEST_F(AttributeUtilityTest, UnknownsUtilityMergeUnknownsFromValues) { AttributeUtility utility(unknown_patterns, missing_attribute_patterns); UnknownValue unknown_set0 = cel::common_internal::MakeUnknownValue( - cel::Unknown(AttributeSet({attribute0}))); + cel::Unknown(AttributeSet({attribute0})), &arena_); UnknownValue unknown_set1 = cel::common_internal::MakeUnknownValue( - cel::Unknown(AttributeSet({attribute1}))); + cel::Unknown(AttributeSet({attribute1})), &arena_); std::vector values = { unknown_set0, @@ -97,7 +98,8 @@ TEST_F(AttributeUtilityTest, UnknownsUtilityMergeUnknownsFromValues) { cel::IntValue(1), }; - absl::optional unknown_set = utility.MergeUnknowns(values); + absl::optional unknown_set = + utility.MergeUnknowns(values, &arena_); ASSERT_TRUE(unknown_set.has_value()); EXPECT_THAT((*unknown_set).ToAttributeSet(), UnorderedPointwise( @@ -161,7 +163,7 @@ TEST_F(AttributeUtilityTest, CreateUnknownSet) { std::vector empty_patterns; AttributeUtility utility(empty_patterns, empty_patterns); - UnknownValue set = utility.CreateUnknownSet(trail.attribute()); + UnknownValue set = utility.CreateUnknownSet(trail.attribute(), &arena_); ASSERT_THAT(set.ToAttributeSet(), SizeIs(1)); ASSERT_OK_AND_ASSIGN(auto elem, set.ToAttributeSet().begin()->AsString()); EXPECT_EQ(elem, "destination.ip"); diff --git a/eval/eval/comprehension_step.cc b/eval/eval/comprehension_step.cc index 321e9c213..3532c1419 100644 --- a/eval/eval/comprehension_step.cc +++ b/eval/eval/comprehension_step.cc @@ -69,7 +69,8 @@ void ComprehensionInitStep::Evaluate(ExecutionFrame* frame) const { const AttributeTrail& top_attr = frame->value_stack().PeekAttribute(); if (frame->attribute_utility().CheckForUnknownPartial(top_attr)) { frame->value_stack().PopAndPush( - frame->attribute_utility().CreateUnknownSet(top_attr.attribute())); + frame->attribute_utility().CreateUnknownSet(top_attr.attribute(), + frame->arena())); frame->JumpToOrAbort(error_jump_offset_); return; } @@ -167,7 +168,7 @@ void ComprehensionNextStep::Evaluate1(ExecutionFrame* frame) const { if (frame->attribute_utility().CheckForUnknownExact( iter_slot->attribute())) { *iter_slot->mutable_value() = frame->attribute_utility().CreateUnknownSet( - iter_slot->attribute().attribute()); + iter_slot->attribute().attribute(), frame->arena()); } } else { absl::StatusOr ok = entry.iterator->Next1( @@ -246,7 +247,7 @@ void ComprehensionNextStep::Evaluate2(ExecutionFrame* frame) const { iter2_slot->attribute())) { *iter2_slot->mutable_value() = frame->attribute_utility().CreateUnknownSet( - iter2_slot->attribute().attribute()); + iter2_slot->attribute().attribute(), frame->arena()); } } } diff --git a/eval/eval/container_access_step.cc b/eval/eval/container_access_step.cc index 5f7cf3e2b..03fcb37d8 100644 --- a/eval/eval/container_access_step.cc +++ b/eval/eval/container_access_step.cc @@ -236,14 +236,15 @@ void PerformLookup(ExecutionFrameBase& frame, const Value& container, unknowns.MaybeAdd(key); if (!unknowns.IsEmpty()) { - result = std::move(unknowns).Build(); + result = std::move(unknowns).Build(frame.arena()); return; } trail = container_trail.Step(AttributeQualifierFromValue(key)); if (frame.attribute_utility().CheckForUnknownExact(trail)) { - result = frame.attribute_utility().CreateUnknownSet(trail.attribute()); + result = frame.attribute_utility().CreateUnknownSet(trail.attribute(), + frame.arena()); return; } } diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index 1d57fafa7..20a9b4f37 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -78,7 +78,7 @@ absl::Status CreateListStep::DoEvaluate(ExecutionFrame* frame, absl::optional unknown_set = frame->attribute_utility().IdentifyAndMergeUnknowns( args, frame->value_stack().GetAttributeSpan(list_size_), - /*use_partial=*/true); + /*use_partial=*/true, frame->arena()); if (unknown_set.has_value()) { *result = std::move(*unknown_set); return absl::OkStatus(); diff --git a/eval/eval/create_map_step.cc b/eval/eval/create_map_step.cc index 8283f1ed8..1b7d994a7 100644 --- a/eval/eval/create_map_step.cc +++ b/eval/eval/create_map_step.cc @@ -72,7 +72,8 @@ absl::StatusOr CreateStructStepForMap::DoEvaluate( if (frame->enable_unknowns()) { absl::optional unknown_set = frame->attribute_utility().IdentifyAndMergeUnknowns( - args, frame->value_stack().GetAttributeSpan(args.size()), true); + args, frame->value_stack().GetAttributeSpan(args.size()), true, + frame->arena()); if (unknown_set.has_value()) { return *unknown_set; } diff --git a/eval/eval/create_struct_step.cc b/eval/eval/create_struct_step.cc index ef05a08f7..5f1aa3f2b 100644 --- a/eval/eval/create_struct_step.cc +++ b/eval/eval/create_struct_step.cc @@ -77,7 +77,7 @@ absl::StatusOr CreateStructStepForStruct::DoEvaluate( absl::optional unknown_set = frame->attribute_utility().IdentifyAndMergeUnknowns( args, frame->value_stack().GetAttributeSpan(entries_size), - /*use_partial=*/true); + /*use_partial=*/true, frame->arena()); if (unknown_set.has_value()) { return *unknown_set; } diff --git a/eval/eval/equality_steps.cc b/eval/eval/equality_steps.cc index 6d8cf5751..5f3a6b58f 100644 --- a/eval/eval/equality_steps.cc +++ b/eval/eval/equality_steps.cc @@ -58,7 +58,7 @@ absl::StatusOr EvaluateEquality( accu.MaybeAdd(lhs, lhs_attr); accu.MaybeAdd(rhs, rhs_attr); if (!accu.IsEmpty()) { - return std::move(accu).Build(); + return std::move(accu).Build(frame.arena()); } } @@ -146,7 +146,7 @@ absl::StatusOr EvaluateIn(ExecutionFrameBase& frame, const Value& item, accu.MaybeAdd(item, item_attr); accu.MaybeAdd(container, container_attr); if (!accu.IsEmpty()) { - return std::move(accu).Build(); + return std::move(accu).Build(frame.arena()); } } if (container.IsList()) { diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 4314f4e38..cbc47d136 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -139,7 +139,7 @@ void ExpressionStep::EvaluateMutableListAppendStep(ExecutionFrame& frame) { std::optional unknown_set = frame.attribute_utility().IdentifyAndMergeUnknowns( args, frame.value_stack().GetAttributeSpan(2), - /*use_partial=*/true); + /*use_partial=*/true, frame.arena()); if (unknown_set.has_value()) { frame.value_stack().PopAndPush(2, std::move(*unknown_set)); return; diff --git a/eval/eval/function_step.cc b/eval/eval/function_step.cc index 4160072fc..37763bc9e 100644 --- a/eval/eval/function_step.cc +++ b/eval/eval/function_step.cc @@ -115,8 +115,8 @@ std::vector CheckForPartialUnknowns( if (frame->attribute_utility().CheckForUnknown(trail, /*use_partial=*/true)) { - result.push_back( - frame->attribute_utility().CreateUnknownSet(trail.attribute())); + result.push_back(frame->attribute_utility().CreateUnknownSet( + trail.attribute(), frame->arena())); } else { result.push_back(args.at(i)); } @@ -161,8 +161,8 @@ inline absl::StatusOr Invoke( if (frame.unknown_function_results_enabled() && IsUnknownFunctionResultError(*result)) { - return frame.attribute_utility().CreateUnknownSet(overload.descriptor, - expr_id, args); + return frame.attribute_utility().CreateUnknownSet( + overload.descriptor, expr_id, args, frame.arena()); } return result; } @@ -184,7 +184,7 @@ Value NoOverloadResult(absl::string_view name, if (frame.unknown_processing_enabled()) { // Already converted partial unknowns to unknown sets so just merge. absl::optional unknown_set = - frame.attribute_utility().MergeUnknowns(args); + frame.attribute_utility().MergeUnknowns(args, frame.arena()); if (unknown_set.has_value()) { return *unknown_set; } diff --git a/eval/eval/ident_step.cc b/eval/eval/ident_step.cc index 0bfbb9211..db1173101 100644 --- a/eval/eval/ident_step.cc +++ b/eval/eval/ident_step.cc @@ -33,8 +33,8 @@ absl::Status LookupIdent(absl::string_view name, ExecutionFrameBase& frame, } if (frame.unknown_processing_enabled() && frame.attribute_utility().CheckForUnknownExact(attribute)) { - result = - frame.attribute_utility().CreateUnknownSet(attribute.attribute()); + result = frame.attribute_utility().CreateUnknownSet(attribute.attribute(), + frame.arena()); return absl::OkStatus(); } } diff --git a/eval/eval/logic_step.cc b/eval/eval/logic_step.cc index 3be16e527..cc0a42cf2 100644 --- a/eval/eval/logic_step.cc +++ b/eval/eval/logic_step.cc @@ -39,7 +39,7 @@ void EvaluateNotStep(ExecutionFrame& frame) { const AttributeTrail& attribute_trail = frame.value_stack().PeekAttribute(); if (frame.attribute_utility().CheckForUnknownPartial(attribute_trail)) { frame.value_stack().PopAndPush(frame.attribute_utility().CreateUnknownSet( - attribute_trail.attribute())); + attribute_trail.attribute(), frame.arena())); return; } } @@ -123,7 +123,7 @@ void EvaluateBoolLogicStep(BoolLogicKind kind, size_t num_args, if (frame.enable_unknowns()) { // Check if unknown? std::optional unknown_set = - frame.attribute_utility().MergeUnknowns(args); + frame.attribute_utility().MergeUnknowns(args, frame.arena()); if (unknown_set.has_value()) { frame.value_stack().PopAndPush(num_args, *std::move(unknown_set)); return; diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 9bfa5fad4..8f58075af 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -55,7 +55,8 @@ absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, ExecutionFrameBase& frame) { if (frame.unknown_processing_enabled() && frame.attribute_utility().CheckForUnknownExact(trail)) { - return frame.attribute_utility().CreateUnknownSet(trail.attribute()); + return frame.attribute_utility().CreateUnknownSet(trail.attribute(), + frame.arena()); } if (frame.missing_attribute_errors_enabled() && diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index 29cb5d6ff..dff02b625 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -688,7 +688,7 @@ absl::StatusOr> CheckForMarkedAttributes( // declared pattern. Follow up will truncate the returned attribute to match // the pattern. return frame.attribute_utility().CreateUnknownSet( - attribute_trail.attribute()); + attribute_trail.attribute(), frame.arena()); } if (frame.missing_attribute_errors_enabled() && From f0050751f4ce4b8b8d6e12294925f1eeaaa6feaa Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Mon, 12 Oct 2026 15:39:42 -0700 Subject: [PATCH 292/350] Tighten validation for comprehension list append and map insert optimizations. PiperOrigin-RevId: 998197901 --- eval/compiler/BUILD | 3 + eval/compiler/flat_expr_builder.cc | 33 ++++--- ...ilder_short_circuiting_conformance_test.cc | 76 ++++++++++++++-- eval/compiler/flat_expr_builder_test.cc | 88 +++++++++++++++++++ 4 files changed, 185 insertions(+), 15 deletions(-) diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index d75e6e50d..7edf542ae 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -151,6 +151,7 @@ cc_test( "//common:kind", "//common:value", "//eval/public:activation", + "//eval/public:ast_rewrite", "//eval/public:builtin_func_registrar", "//eval/public:cel_attribute", "//eval/public:cel_expr_builder_factory", @@ -161,6 +162,7 @@ cc_test( "//eval/public:cel_options", "//eval/public:cel_value", "//eval/public:portable_cel_function_adapter", + "//eval/public:source_position", "//eval/public:unknown_attribute_set", "//eval/public:unknown_set", "//eval/public/containers:container_backed_map_impl", @@ -470,6 +472,7 @@ cc_test( deps = [ ":cel_expression_builder_flat_impl", "//eval/public:activation", + "//eval/public:builtin_func_registrar", "//eval/public:cel_attribute", "//eval/public:cel_expression", "//eval/public:cel_value", diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index a305dec17..c764b22eb 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -282,12 +282,15 @@ size_t SizeHint(const cel::Expr& expr) { // macro implementation. It is not exhaustive, so it is unsafe to use with // custom comprehensions outside of the standard macros or hand crafted ASTs. bool IsOptimizableListAppend(const cel::ComprehensionExpr* comprehension, - bool enable_comprehension_list_append) { + bool enable_comprehension_list_append, + bool short_circuiting = true) { if (!enable_comprehension_list_append) { return false; } absl::string_view accu_var = comprehension->accu_var(); - if (accu_var.empty() || + if (accu_var.empty() || !absl::StartsWith(accu_var, "@") || + !comprehension->has_result() || + !comprehension->result().has_ident_expr() || comprehension->result().ident_expr().name() != accu_var) { return false; } @@ -308,7 +311,9 @@ bool IsOptimizableListAppend(const cel::ComprehensionExpr* comprehension, if (call_expr->function() == cel::builtin::kTernary && call_expr->args().size() == 3) { - if (!call_expr->args()[1].has_call_expr()) { + if (!short_circuiting || !call_expr->args()[1].has_call_expr() || + !call_expr->args()[2].has_ident_expr() || + call_expr->args()[2].ident_expr().name() != accu_var) { return false; } call_expr = &(call_expr->args()[1].call_expr()); @@ -319,7 +324,8 @@ bool IsOptimizableListAppend(const cel::ComprehensionExpr* comprehension, call_expr->args()[0].has_ident_expr() && call_expr->args()[0].ident_expr().name() == accu_var && call_expr->args()[1].has_list_expr() && - call_expr->args()[1].list_expr().elements().size() == 1; + call_expr->args()[1].list_expr().elements().size() == 1 && + !call_expr->args()[1].list_expr().elements()[0].optional(); } // Assuming `IsOptimizableListAppend()` return true, return a pointer to the @@ -353,7 +359,8 @@ const cel::Expr* GetOptimizableListAppendOperand( // map transformations. It is not exhaustive, so it is unsafe to use with custom // comprehensions outside of the standard macros or hand crafted ASTs. bool IsOptimizableMapInsert(const cel::ComprehensionExpr* comprehension, - bool enable_comprehension_mutable_map) { + bool enable_comprehension_mutable_map, + bool short_circuiting = true) { if (!enable_comprehension_mutable_map) { return false; } @@ -361,12 +368,14 @@ bool IsOptimizableMapInsert(const cel::ComprehensionExpr* comprehension, return false; } absl::string_view accu_var = comprehension->accu_var(); - if (accu_var.empty() || !comprehension->has_result() || + if (accu_var.empty() || !absl::StartsWith(accu_var, "@") || + !comprehension->has_result() || !comprehension->result().has_ident_expr() || comprehension->result().ident_expr().name() != accu_var) { return false; } - if (!comprehension->accu_init().has_map_expr()) { + if (!comprehension->accu_init().has_map_expr() || + !comprehension->accu_init().map_expr().entries().empty()) { return false; } if (!comprehension->loop_step().has_call_expr()) { @@ -376,7 +385,9 @@ bool IsOptimizableMapInsert(const cel::ComprehensionExpr* comprehension, if (call_expr->function() == cel::builtin::kTernary && call_expr->args().size() == 3) { - if (!call_expr->args()[1].has_call_expr()) { + if (!short_circuiting || !call_expr->args()[1].has_call_expr() || + !call_expr->args()[2].has_ident_expr() || + call_expr->args()[2].ident_expr().name() != accu_var) { return false; } call_expr = &(call_expr->args()[1].call_expr()); @@ -1141,10 +1152,12 @@ class FlatExprVisitor : public cel::AstVisitor { /*subexpression=*/-1, /*.is_optimizable_list_append=*/ IsOptimizableListAppend(&comprehension, - options_.enable_comprehension_list_append), + options_.enable_comprehension_list_append, + options_.short_circuiting), /*.is_optimizable_map_insert=*/ IsOptimizableMapInsert(&comprehension, - options_.enable_comprehension_mutable_map), + options_.enable_comprehension_mutable_map, + options_.short_circuiting), /*.is_optimizable_bind=*/is_bind, /*.iter_var_in_scope=*/false, /*.iter_var2_in_scope=*/false, diff --git a/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc b/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc index 641ca9cd1..9081b4813 100644 --- a/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc +++ b/eval/compiler/flat_expr_builder_short_circuiting_conformance_test.cc @@ -11,6 +11,7 @@ #include "absl/strings/string_view.h" #include "eval/compiler/cel_expression_builder_flat_impl.h" #include "eval/public/activation.h" +#include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_expression.h" #include "eval/public/cel_value.h" @@ -37,13 +38,12 @@ using ::testing::SizeIs; void BuildAndEval(CelExpressionBuilder* builder, const Expr& expr, const Activation& activation, google::protobuf::Arena* arena, CelValue* result) { - ASSERT_OK_AND_ASSIGN(auto expression, + ASSERT_OK_AND_ASSIGN(std::unique_ptr expression, builder->CreateExpression(&expr, nullptr)); - auto value = expression->Evaluate(activation, arena); - ASSERT_OK(value); + ASSERT_OK_AND_ASSIGN(CelValue value, expression->Evaluate(activation, arena)); - *result = *value; + *result = value; } class ShortCircuitingTest @@ -53,15 +53,18 @@ class ShortCircuitingTest bool enable_variadic() const { return std::get<1>(GetParam()); } std::unique_ptr GetBuilder( - bool enable_unknowns = false) { + bool enable_unknowns = false, + bool enable_comprehension_list_append = false) { cel::RuntimeOptions options; options.short_circuiting = short_circuiting(); + options.enable_comprehension_list_append = enable_comprehension_list_append; if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeAndFunction; } auto result = std::make_unique( NewTestingRuntimeEnv(), options); + ABSL_CHECK_OK(RegisterBuiltinFunctions(result->GetRegistry())); return result; } @@ -419,6 +422,69 @@ TEST_P(ShortCircuitingTest, TernaryUnknownAndErrorHandling) { EXPECT_EQ(attrs.begin()->variable_name(), "cond"); } +TEST_P(ShortCircuitingTest, FilterComprehension) { + Expr expr = ParseExpr("[1, 2, 3, 4].filter(x, x > 2) == [3, 4]"); + Expr empty_expr = ParseExpr("[1, 2, 3].filter(x, false) == []"); + Activation activation; + google::protobuf::Arena arena; + + for (bool enable_list_append : {false, true}) { + std::unique_ptr builder = + GetBuilder(/*enable_unknowns=*/false, enable_list_append); + + CelValue result; + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), empty_expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + } +} + +TEST_P(ShortCircuitingTest, MapWithFilterComprehension) { + Expr expr = ParseExpr("[1, 2, 3, 4].map(x, x % 2 == 1, x * 10) == [10, 30]"); + Expr empty_expr = ParseExpr("[1, 2, 3].map(x, false, x * 10) == []"); + Activation activation; + google::protobuf::Arena arena; + + for (bool enable_list_append : {false, true}) { + std::unique_ptr builder = + GetBuilder(/*enable_unknowns=*/false, enable_list_append); + + CelValue result; + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), empty_expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + } +} + +TEST_P(ShortCircuitingTest, MapComprehension) { + Expr expr = ParseExpr("[1, 2, 3].map(x, x * 10) == [10, 20, 30]"); + Activation activation; + google::protobuf::Arena arena; + + for (bool enable_list_append : {false, true}) { + std::unique_ptr builder = + GetBuilder(/*enable_unknowns=*/false, enable_list_append); + + CelValue result; + ASSERT_NO_FATAL_FAILURE( + BuildAndEval(builder.get(), expr, activation, &arena, &result)); + ASSERT_TRUE(result.IsBool()) << result.DebugString(); + EXPECT_TRUE(result.BoolOrDie()); + } +} + std::string TestName(testing::TestParamInfo> info) { return absl::StrCat( std::get<0>(info.param) ? "short_circuit_enabled" diff --git a/eval/compiler/flat_expr_builder_test.cc b/eval/compiler/flat_expr_builder_test.cc index 8611de388..f9d302051 100644 --- a/eval/compiler/flat_expr_builder_test.cc +++ b/eval/compiler/flat_expr_builder_test.cc @@ -42,6 +42,7 @@ #include "eval/compiler/constant_folding.h" #include "eval/compiler/qualified_reference_resolver.h" #include "eval/public/activation.h" +#include "eval/public/ast_rewrite.h" #include "eval/public/builtin_func_registrar.h" #include "eval/public/cel_attribute.h" #include "eval/public/cel_expr_builder_factory.h" @@ -53,6 +54,7 @@ #include "eval/public/cel_value.h" #include "eval/public/containers/container_backed_map_impl.h" #include "eval/public/portable_cel_function_adapter.h" +#include "eval/public/source_position.h" #include "eval/public/structs/cel_proto_descriptor_pool_builder.h" #include "eval/public/structs/cel_proto_wrapper.h" #include "eval/public/testing/matchers.h" @@ -3016,6 +3018,92 @@ INSTANTIATE_TEST_SUITE_P( VariadicLogicalEvalTestCase{"All_Unknown", "[a, b, c].all(x, x)", "true", "unknown1", "true", "unknown"})); +void ReplaceResultAccumulatorWithLegacyName(Expr* expr) { + class LegacyAccumulatorRewriter : public AstRewriterBase { + public: + bool PostVisitRewrite(Expr* expr, const SourcePosition*) override { + if (expr->has_ident_expr() && expr->ident_expr().name() == "@result") { + expr->mutable_ident_expr()->set_name("__result__"); + return true; + } + if (expr->has_comprehension_expr() && + expr->comprehension_expr().accu_var() == "@result") { + expr->mutable_comprehension_expr()->set_accu_var("__result__"); + return true; + } + return false; + } + } rewriter; + AstRewrite(expr, /*source_info=*/nullptr, &rewriter); +} + +TEST(FlatExprBuilderTest, LegacyAccumulatorReferenceDoesNotMutateInPlace) { + cel::RuntimeOptions options; + options.enable_comprehension_list_append = true; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + for (absl::string_view expr_str : { + "[1, 2].map(x, (__result__ + [x]).size()) == [1, 2]", + "[1, 2, 3].filter(x, (__result__ + [x]).size() > 1) == []", + "[1, 2, 3].filter(x, (__result__ + [x]).size() == 1) == [1]", + }) { + ASSERT_OK_AND_ASSIGN(ParsedExpr parsed_expr, parser::Parse(expr_str)); + ReplaceResultAccumulatorWithLegacyName(parsed_expr.mutable_expr()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + Activation activation; + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(CelValue result, + cel_expr->Evaluate(activation, &arena)); + EXPECT_THAT(result, test::IsCelBool(true)) << expr_str; + } +} + +TEST(FlatExprBuilderTest, + ComprehensionTernaryNonIdentFalseBranchSkipsMutableListAppend) { + // Hand-crafted comprehension where the false branch of the ternary loop_step + // is not the accumulator variable: + // accu_var = "__result__", accu_init = [] + // loop_step = x > 1 ? (__result__ + [x]) : [0] + // For iter_range = [1, 2, 3]: + // x = 1 -> false branch -> __result__ becomes [0] + // x = 2 -> true branch -> __result__ becomes [0, 2] + // x = 3 -> true branch -> __result__ becomes [0, 2, 3] + ASSERT_OK_AND_ASSIGN( + ParsedExpr parsed_expr, + parser::Parse("[1, 2, 3].filter(x, x > 1) == [0, 2, 3]")); + Expr* filter_expr = + parsed_expr.mutable_expr()->mutable_call_expr()->mutable_args(0); + ASSERT_TRUE(filter_expr->has_comprehension_expr()); + Expr* false_branch = filter_expr->mutable_comprehension_expr() + ->mutable_loop_step() + ->mutable_call_expr() + ->mutable_args(2); + false_branch->Clear(); + false_branch->mutable_list_expr() + ->add_elements() + ->mutable_const_expr() + ->set_int64_value(0); + + cel::RuntimeOptions options; + options.enable_comprehension_list_append = true; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr cel_expr, + builder.CreateExpression(&parsed_expr.expr(), + &parsed_expr.source_info())); + + Activation activation; + google::protobuf::Arena arena; + ASSERT_OK_AND_ASSIGN(CelValue result, cel_expr->Evaluate(activation, &arena)); + EXPECT_THAT(result, test::IsCelBool(true)); +} + } // namespace } // namespace google::api::expr::runtime From 21a5734c805b086a5f4241471c9201bf45a7bea2 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 13 Oct 2026 08:41:57 -0700 Subject: [PATCH 293/350] Unwrap compat and parsed legacy list/map wrappers in `FromLegacyList` and `FromLegacyMap`, and avoid per-call scratch arenas in legacy map `Has`. 1. Add `FromLegacyList` and `FromLegacyMap` (used by `ModernValue` and `FromLegacyValue`) to unwrap `LegacyParsedRepeatedFieldListValue`, `LegacyParsedJsonListValue`, `CompatListValue`, and `MutableCompatListValue` (and their map counterparts `LegacyParsedMapFieldMapValue`, `LegacyParsedJsonMapValue`, `CompatMapValue`, and `MutableCompatMapValue`) back into `CustomListValue` / `CustomMapValue` using their owning arena instead of re-wrapping them in `LegacyListValue` / `LegacyMapValue`. 2. In `LegacyParsedMapFieldMapValue::Has(const CelValue&)` and `LegacyParsedJsonMapValue::Has(const CelValue&)`, use `arena_` instead of constructing a per-call `proto2::Arena scratch_arena`. PiperOrigin-RevId: 998661512 --- common/legacy_value.cc | 11 +++---- common/values/legacy_list_value.cc | 39 ++++++++++++++++++++++++ common/values/legacy_list_value.h | 3 ++ common/values/legacy_map_value.cc | 48 ++++++++++++++++++++++++++---- common/values/legacy_map_value.h | 3 ++ 5 files changed, 91 insertions(+), 13 deletions(-) diff --git a/common/legacy_value.cc b/common/legacy_value.cc index 622257f43..151d848af 100644 --- a/common/legacy_value.cc +++ b/common/legacy_value.cc @@ -1063,12 +1063,10 @@ absl::Status ModernValue(google::protobuf::Arena* arena, result = UnsafeTimestampValue(legacy_value.TimestampOrDie()); return absl::OkStatus(); case CelValue::Type::kList: - result = - ListValue(common_internal::LegacyListValue(legacy_value.ListOrDie())); + result = common_internal::FromLegacyList(legacy_value.ListOrDie()); return absl::OkStatus(); case CelValue::Type::kMap: - result = - MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); + result = common_internal::FromLegacyMap(legacy_value.MapOrDie()); return absl::OkStatus(); case CelValue::Type::kUnknownSet: result = common_internal::MakeUnknownValue( @@ -1178,10 +1176,9 @@ absl::StatusOr FromLegacyValue(google::protobuf::Arena* arena, case CelValue::Type::kTimestamp: return UnsafeTimestampValue(legacy_value.TimestampOrDie()); case CelValue::Type::kList: - return ListValue( - common_internal::LegacyListValue(legacy_value.ListOrDie())); + return common_internal::FromLegacyList(legacy_value.ListOrDie()); case CelValue::Type::kMap: - return MapValue(common_internal::LegacyMapValue(legacy_value.MapOrDie())); + return common_internal::FromLegacyMap(legacy_value.MapOrDie()); case CelValue::Type::kUnknownSet: return common_internal::MakeUnknownValue(*legacy_value.UnknownSetOrDie(), arena); diff --git a/common/values/legacy_list_value.cc b/common/values/legacy_list_value.cc index d3c1b94be..50bf1e6fe 100644 --- a/common/values/legacy_list_value.cc +++ b/common/values/legacy_list_value.cc @@ -80,6 +80,8 @@ class LegacyParsedRepeatedFieldListValue final return UnsafeLegacyValue(result, /*stable=*/false, arena); } + google::protobuf::Arena* absl_nonnull arena() const { return arena_; } + protected: std::string DebugString() const override { return value_.DebugString(); } @@ -202,6 +204,8 @@ class LegacyParsedJsonListValue final return UnsafeLegacyValue(result, /*stable=*/false, arena); } + google::protobuf::Arena* absl_nonnull arena() const { return arena_; } + protected: std::string DebugString() const override { return value_.DebugString(); } @@ -296,6 +300,41 @@ CustomListValue WrapLegacyParsedJsonList(ParsedJsonListValue value, arena); } +ListValue FromLegacyList( + const google::api::expr::runtime::CelList* absl_nullable cel_list) { + if (cel_list != nullptr) { + NativeTypeId native_type_id = NativeTypeId::Of(*cel_list); + if (native_type_id == + NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast( + cel_list); + return CustomListValue(impl, impl->arena()); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_list); + return CustomListValue(impl, impl->arena()); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_list); + if (google::protobuf::Arena* arena = impl->GetOwningArena(); arena != nullptr) { + return CustomListValue(impl, arena); + } + return CustomListValue(); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_list); + if (google::protobuf::Arena* arena = impl->GetOwningArena(); arena != nullptr) { + return CustomListValue(impl, arena); + } + } + } + return LegacyListValue(cel_list); +} + absl::Status LegacyListValue::Equal( const Value& other, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, diff --git a/common/values/legacy_list_value.h b/common/values/legacy_list_value.h index 6dd6a568e..55a169653 100644 --- a/common/values/legacy_list_value.h +++ b/common/values/legacy_list_value.h @@ -171,6 +171,9 @@ CustomListValue WrapLegacyParsedRepeatedField( CustomListValue WrapLegacyParsedJsonList(ParsedJsonListValue value, google::protobuf::Arena* absl_nonnull arena); +ListValue FromLegacyList( + const google::api::expr::runtime::CelList* absl_nullable cel_list); + } // namespace common_internal } // namespace cel diff --git a/common/values/legacy_map_value.cc b/common/values/legacy_map_value.cc index 8d9ce2ad3..bd025c51b 100644 --- a/common/values/legacy_map_value.cc +++ b/common/values/legacy_map_value.cc @@ -253,11 +253,10 @@ class LegacyParsedMapFieldMapValue final const google::api::expr::runtime::CelValue& key) const override { CEL_RETURN_IF_ERROR( google::api::expr::runtime::CelValue::CheckMapKeyType(key)); - google::protobuf::Arena scratch_arena; Value modern_key; - CEL_RETURN_IF_ERROR(ModernValue(&scratch_arena, key, modern_key)); + CEL_RETURN_IF_ERROR(ModernValue(arena_, key, modern_key)); return Has(modern_key, google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), &scratch_arena); + google::protobuf::MessageFactory::generated_factory(), arena_); } absl::StatusOr ListKeys() @@ -281,6 +280,8 @@ class LegacyParsedMapFieldMapValue final return legacy_list->cel_list(); } + google::protobuf::Arena* absl_nonnull arena() const { return arena_; } + private: NativeTypeId GetNativeTypeId() const override { return NativeTypeId::For(); @@ -440,11 +441,10 @@ class LegacyParsedJsonMapValue final const google::api::expr::runtime::CelValue& key) const override { CEL_RETURN_IF_ERROR( google::api::expr::runtime::CelValue::CheckMapKeyType(key)); - google::protobuf::Arena scratch_arena; Value modern_key; - CEL_RETURN_IF_ERROR(ModernValue(&scratch_arena, key, modern_key)); + CEL_RETURN_IF_ERROR(ModernValue(arena_, key, modern_key)); return Has(modern_key, google::protobuf::DescriptorPool::generated_pool(), - google::protobuf::MessageFactory::generated_factory(), &scratch_arena); + google::protobuf::MessageFactory::generated_factory(), arena_); } absl::StatusOr ListKeys() @@ -468,6 +468,8 @@ class LegacyParsedJsonMapValue final return legacy_list->cel_list(); } + google::protobuf::Arena* absl_nonnull arena() const { return arena_; } + private: NativeTypeId GetNativeTypeId() const override { return NativeTypeId::For(); @@ -491,6 +493,40 @@ CustomMapValue WrapLegacyParsedJsonMap(ParsedJsonMapValue value, arena); } +MapValue FromLegacyMap( + const google::api::expr::runtime::CelMap* absl_nullable cel_map) { + if (cel_map != nullptr) { + NativeTypeId native_type_id = NativeTypeId::Of(*cel_map); + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast( + cel_map); + return CustomMapValue(impl, impl->arena()); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_map); + return CustomMapValue(impl, impl->arena()); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_map); + if (google::protobuf::Arena* arena = impl->GetOwningArena(); arena != nullptr) { + return CustomMapValue(impl, arena); + } + return CustomMapValue(); + } + if (native_type_id == NativeTypeId::For()) { + const auto* impl = + cel::internal::down_cast(cel_map); + if (google::protobuf::Arena* arena = impl->GetOwningArena(); arena != nullptr) { + return CustomMapValue(impl, arena); + } + } + } + return LegacyMapValue(cel_map); +} + absl::Status LegacyMapValue::Equal( const Value& other, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, diff --git a/common/values/legacy_map_value.h b/common/values/legacy_map_value.h index 53f39a8de..c170dea80 100644 --- a/common/values/legacy_map_value.h +++ b/common/values/legacy_map_value.h @@ -188,6 +188,9 @@ CustomMapValue WrapLegacyParsedMapField(ParsedMapFieldValue value, CustomMapValue WrapLegacyParsedJsonMap(ParsedJsonMapValue value, google::protobuf::Arena* absl_nonnull arena); +MapValue FromLegacyMap( + const google::api::expr::runtime::CelMap* absl_nullable cel_map); + } // namespace common_internal } // namespace cel From 1ae715dcfa2f61a87cdf422cf4cc3be9c2bc52df Mon Sep 17 00:00:00 2001 From: Justin King Date: Tue, 13 Oct 2026 09:45:25 -0700 Subject: [PATCH 294/350] Take `cel::AttributeMatch` by value PiperOrigin-RevId: 998699200 --- base/attribute_matcher.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/base/attribute_matcher.h b/base/attribute_matcher.h index 01d60ba85..f10448b0b 100644 --- a/base/attribute_matcher.h +++ b/base/attribute_matcher.h @@ -200,8 +200,7 @@ AttributeMatch MakeAttributeMatch( const common_internal::AttributeMatcherNode* node); [[nodiscard]] -const AttributeMatcherNode* GetAttributeMatcherNode( - const AttributeMatch& match); +const AttributeMatcherNode* GetAttributeMatcherNode(AttributeMatch match); } // namespace common_internal @@ -252,7 +251,7 @@ class [[nodiscard]] AttributeMatch { friend AttributeMatch common_internal::MakeAttributeMatch( const common_internal::AttributeMatcherNode* node); friend const common_internal::AttributeMatcherNode* - common_internal::GetAttributeMatcherNode(const AttributeMatch& match); + common_internal::GetAttributeMatcherNode(AttributeMatch match); AttributeMatch NotNoneMatchQualifier( const AttributeQualifierView& qualifier) const { @@ -276,7 +275,7 @@ inline AttributeMatch MakeAttributeMatch( [[nodiscard]] inline const AttributeMatcherNode* GetAttributeMatcherNode( - const AttributeMatch& match) { + AttributeMatch match) { return match.node_; } From 18cc01bc450923f6bca78ced3a848be92cd4c40e Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 13 Oct 2026 11:33:01 -0700 Subject: [PATCH 295/350] Track iterator stack limit separately from comprehension slots. PiperOrigin-RevId: 998777471 --- eval/compiler/constant_folding.cc | 5 +-- eval/compiler/flat_expr_builder.cc | 52 ++++++++++++++++++++++++++++-- eval/eval/BUILD | 1 + eval/eval/equality_steps_test.cc | 8 ++--- eval/eval/evaluator_core.cc | 8 ++--- eval/eval/evaluator_core.h | 21 ++++++------ eval/eval/evaluator_core_test.cc | 4 +-- eval/eval/iterator_stack.h | 3 +- eval/eval/lazy_init_step_test.cc | 3 +- 9 files changed, 77 insertions(+), 28 deletions(-) diff --git a/eval/compiler/constant_folding.cc b/eval/compiler/constant_folding.cc index fb58d7d48..4a8d55c31 100644 --- a/eval/compiler/constant_folding.cc +++ b/eval/compiler/constant_folding.cc @@ -75,8 +75,9 @@ class ConstantFoldingExtension : public ProgramOptimizer { const TypeProvider& type_provider) : shared_arena_(std::move(shared_arena)), shared_message_factory_(std::move(shared_message_factory)), - state_(kDefaultStackLimit, kComprehensionSlotCount, type_provider, - descriptor_pool, message_factory, arena) {} + state_(kDefaultStackLimit, kComprehensionSlotCount, + /*iterator_stack_size=*/0, type_provider, descriptor_pool, + message_factory, arena) {} absl::Status OnPreVisit(google::api::expr::runtime::PlannerContext& context, const Expr& node) override; diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index c764b22eb..fa0d556b6 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -105,7 +105,12 @@ class FlatExprVisitor; // Helper for bookkeeping variables mapped to indexes. class IndexManager { public: - IndexManager() : next_free_slot_(0), max_slot_count_(0) {} + IndexManager() + : next_free_slot_(0), + max_slot_count_(0), + iter_stack_count_(0), + max_iter_stack_count_(0), + lazy_iter_stack_count_(0) {} size_t ReserveSlots(size_t n) { size_t result = next_free_slot_; @@ -121,11 +126,30 @@ class IndexManager { return next_free_slot_; } + void ReserveIterStack() { + ++iter_stack_count_; + max_iter_stack_count_ = std::max(max_iter_stack_count_, iter_stack_count_); + } + void ReleaseIterStack() { --iter_stack_count_; } + void AddLazyIterStack() { ++lazy_iter_stack_count_; } + size_t max_slot_count() const { return max_slot_count_; } + // Returns the maximum number of iterator stack slots required to evaluate the + // expression by tracking the maximum comprehension nesting in the main + // expression and dedicating a slot for any comprehensions evaluated lazily. + // This is not as tight as possible, but is a correct upper bound estimate and + // is tight for small expressions. + size_t IterStackRequirement() const { + return max_iter_stack_count_ + lazy_iter_stack_count_; + } + private: size_t next_free_slot_; size_t max_slot_count_; + size_t iter_stack_count_; + size_t max_iter_stack_count_; + size_t lazy_iter_stack_count_; }; // Helper for computing jump offsets. @@ -1132,21 +1156,34 @@ class FlatExprVisitor : public cel::AstVisitor { slot_count = 0; } } + // If this is in the scope of an optimized bind accu-init, account the slots // to the outermost bind-init scope. // // The init expression is effectively inlined at the first usage in the // critical path (which is unknown at plan time), so the used slots need to // be dedicated for the entire scope of that bind. + bool in_bind_init = false; for (ComprehensionStackRecord& record : comprehension_stack_) { if (record.in_accu_init && record.is_optimizable_bind) { record.slot_count += slot_count; slot_count = 0; + in_bind_init = true; break; } // If no bind init subexpression, account normally. } + const bool is_stack_releasable = + !in_bind_init && (!block_.has_value() || !block_->in); + if (!is_bind) { + if (is_stack_releasable) { + index_manager_.AddLazyIterStack(); + } else { + index_manager_.ReserveIterStack(); + } + } + comprehension_stack_.push_back( {&expr, &comprehension, iter_slot, iter2_slot, accu_slot, slot_count, /*subexpression=*/-1, @@ -1159,6 +1196,7 @@ class FlatExprVisitor : public cel::AstVisitor { options_.enable_comprehension_mutable_map, options_.short_circuiting), /*.is_optimizable_bind=*/is_bind, + /*.is_stack_releasable=*/is_stack_releasable, /*.iter_var_in_scope=*/false, /*.iter_var2_in_scope=*/false, /*.accu_var_in_scope=*/false, @@ -1185,6 +1223,9 @@ class FlatExprVisitor : public cel::AstVisitor { record.visitor->PostVisit(&expr); index_manager_.ReleaseSlots(record.slot_count); + if (record.is_stack_releasable) { + index_manager_.ReleaseIterStack(); + } comprehension_stack_.pop_back(); } @@ -1476,6 +1517,10 @@ class FlatExprVisitor : public cel::AstVisitor { size_t slot_count() const { return index_manager_.max_slot_count(); } + size_t iter_stack_count() const { + return index_manager_.IterStackRequirement(); + } + void AddOptimizer(std::unique_ptr optimizer) { program_optimizers_.push_back(std::move(optimizer)); } @@ -1507,6 +1552,7 @@ class FlatExprVisitor : public cel::AstVisitor { bool is_optimizable_list_append; bool is_optimizable_map_insert; bool is_optimizable_bind; + bool is_stack_releasable; bool iter_var_in_scope; bool iter_var2_in_scope; bool accu_var_in_scope; @@ -2275,8 +2321,8 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( FlattenExpressionTable(program_builder, execution_path); return FlatExpression(std::move(execution_path), std::move(subexpressions), - visitor.slot_count(), GetTypeProvider(), options_, - std::move(arena)); + visitor.slot_count(), visitor.iter_stack_count(), + GetTypeProvider(), options_, std::move(arena)); } const cel::TypeProvider& FlatExprBuilder::GetTypeProvider() const { return use_legacy_type_provider_ diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 08dacb9b2..f505954bf 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -949,6 +949,7 @@ cc_library( deps = [ "//common:value", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:inlined_vector", "@com_google_absl//absl/log:absl_check", ], ) diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index 4ebfe10ef..50970ca44 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -81,7 +81,7 @@ TEST(IterativeTest, PartialAttrUnknown) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); @@ -113,7 +113,7 @@ TEST(IterativeTest, PartialAttrUnknownDisabled) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); @@ -183,7 +183,7 @@ TEST_P(EqualsTest, Iterative) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); @@ -328,7 +328,7 @@ TEST_P(InTest, Iterative) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index cbc47d136..2fc6eda93 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -222,16 +222,16 @@ FlatExpressionEvaluatorState FlatExpression::MakeEvaluatorState( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { return FlatExpressionEvaluatorState(path_.size(), comprehension_slots_size_, - type_provider_, descriptor_pool, - message_factory, arena); + iterator_stack_size_, type_provider_, + descriptor_pool, message_factory, arena); } FlatExpressionEvaluatorState FlatExpression::MakeEvaluatorState( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory) const { return FlatExpressionEvaluatorState(path_.size(), comprehension_slots_size_, - type_provider_, descriptor_pool, - message_factory); + iterator_stack_size_, type_provider_, + descriptor_pool, message_factory); } absl::StatusOr FlatExpression::EvaluateWithCallback( diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index d23dfd045..07d985804 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -430,14 +430,11 @@ class FlatExpressionEvaluatorState { public: FlatExpressionEvaluatorState( size_t value_stack_size, size_t comprehension_slot_count, - const cel::TypeProvider& type_provider, + size_t iterator_stack_size, const cel::TypeProvider& type_provider, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory = nullptr) : value_stack_(value_stack_size), - // We currently use comprehension_slot_count because it is less of an - // over estimate than value_stack_size. In future we should just - // calculate the correct capacity. - iterator_stack_(comprehension_slot_count), + iterator_stack_(iterator_stack_size), comprehension_slots_(comprehension_slot_count), type_provider_(type_provider), descriptor_pool_(descriptor_pool), @@ -445,15 +442,12 @@ class FlatExpressionEvaluatorState { FlatExpressionEvaluatorState( size_t value_stack_size, size_t comprehension_slot_count, - const cel::TypeProvider& type_provider, + size_t iterator_stack_size, const cel::TypeProvider& type_provider, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) : value_stack_(value_stack_size), - // We currently use comprehension_slot_count because it is less of an - // over estimate than value_stack_size. In future we should just - // calculate the correct capacity. - iterator_stack_(comprehension_slot_count), + iterator_stack_(iterator_stack_size), comprehension_slots_(comprehension_slot_count), type_provider_(type_provider), descriptor_pool_(descriptor_pool), @@ -859,19 +853,21 @@ class FlatExpression { : path_(std::move(path)), subexpressions_({path_}), comprehension_slots_size_(comprehension_slots_size), + iterator_stack_size_(comprehension_slots_size), type_provider_(type_provider), options_(options), arena_(std::move(arena)) {} FlatExpression(ExecutionPath path, std::vector subexpressions, - size_t comprehension_slots_size, + size_t comprehension_slots_size, size_t iterator_stack_size, const cel::TypeProvider& type_provider, const cel::RuntimeOptions& options, absl_nullable std::shared_ptr arena = nullptr) : path_(std::move(path)), subexpressions_(std::move(subexpressions)), comprehension_slots_size_(comprehension_slots_size), + iterator_stack_size_(iterator_stack_size), type_provider_(type_provider), options_(options), arena_(std::move(arena)) {} @@ -914,12 +910,15 @@ class FlatExpression { size_t comprehension_slots_size() const { return comprehension_slots_size_; } + size_t iterator_stack_size() const { return iterator_stack_size_; } + const cel::TypeProvider& type_provider() const { return type_provider_; } private: ExecutionPath path_; std::vector subexpressions_; size_t comprehension_slots_size_; + size_t iterator_stack_size_; const cel::TypeProvider& type_provider_; cel::RuntimeOptions options_; // Arena used during planning phase, may hold constant values so should be diff --git a/eval/eval/evaluator_core_test.cc b/eval/eval/evaluator_core_test.cc index c67dc8b58..1f43f3f35 100644 --- a/eval/eval/evaluator_core_test.cc +++ b/eval/eval/evaluator_core_test.cc @@ -84,7 +84,7 @@ TEST(EvaluatorCoreTest, ExecutionFrameNext) { cel::Activation activation; FlatExpressionEvaluatorState state( path.size(), - /*comprehension_slots_size=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); ExecutionFrame frame(path, activation, options, state); @@ -111,7 +111,7 @@ TEST(EvaluatorCoreTest, JumpOutOfRangeAborts) { cel::Activation activation; FlatExpressionEvaluatorState state( path.size(), - /*comprehension_slots_size=*/0, type_provider, + /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); ExecutionFrame frame(path, activation, options, state); diff --git a/eval/eval/iterator_stack.h b/eval/eval/iterator_stack.h index 0b0de14aa..ccb377dd3 100644 --- a/eval/eval/iterator_stack.h +++ b/eval/eval/iterator_stack.h @@ -20,6 +20,7 @@ #include #include "absl/base/nullability.h" +#include "absl/container/inlined_vector.h" #include "absl/log/absl_check.h" #include "common/value.h" @@ -92,7 +93,7 @@ class IteratorStack final { } private: - std::vector entries_; + absl::InlinedVector entries_; size_t max_size_; }; diff --git a/eval/eval/lazy_init_step_test.cc b/eval/eval/lazy_init_step_test.cc index 3b5282d21..d7211e597 100644 --- a/eval/eval/lazy_init_step_test.cc +++ b/eval/eval/lazy_init_step_test.cc @@ -46,7 +46,8 @@ class LazyInitStepTest : public testing::Test { public: LazyInitStepTest() : type_provider_(cel::internal::GetTestingDescriptorPool()), - evaluator_state_(kValueStack, kComprehensionSlotCount, type_provider_, + evaluator_state_(kValueStack, kComprehensionSlotCount, + /*iterator_stack_size=*/0, type_provider_, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena_) {} From 18789ecaf8cd193d8d060f76c2b50056cc8a4988 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 13 Oct 2026 13:39:52 -0700 Subject: [PATCH 296/350] Add instruction for list steps. PiperOrigin-RevId: 998861772 --- eval/compiler/BUILD | 2 - eval/compiler/constant_folding_test.cc | 12 +- eval/compiler/flat_expr_builder.cc | 23 +++- eval/eval/BUILD | 28 +---- eval/eval/create_list_step.cc | 159 ++++++++++++++----------- eval/eval/create_list_step.h | 44 ++++++- eval/eval/create_list_step_test.cc | 12 +- eval/eval/evaluator_core.h | 30 +++++ 8 files changed, 189 insertions(+), 121 deletions(-) diff --git a/eval/compiler/BUILD b/eval/compiler/BUILD index 7edf542ae..8d9a5aa3b 100644 --- a/eval/compiler/BUILD +++ b/eval/compiler/BUILD @@ -102,7 +102,6 @@ cc_library( "//common:type_spec_resolver", "//common:value", "//eval/eval:container_access_step", - "//eval/eval:create_list_step", "//eval/eval:create_map_step", "//eval/eval:create_struct_step", "//eval/eval:evaluator_core", @@ -333,7 +332,6 @@ cc_test( "//base:ast", "//common:expr", "//common:value", - "//eval/eval:create_list_step", "//eval/eval:create_map_step", "//eval/eval:evaluator_core", "//extensions/protobuf:ast_converters", diff --git a/eval/compiler/constant_folding_test.cc b/eval/compiler/constant_folding_test.cc index 8c6fcf54c..d9ba91bfa 100644 --- a/eval/compiler/constant_folding_test.cc +++ b/eval/compiler/constant_folding_test.cc @@ -298,10 +298,8 @@ TEST_F(UpdatedConstantFoldingTest, CreatesList) { program_builder.ExitSubexpression(&elem_two); // createlist - ASSERT_OK_AND_ASSIGN(auto step, - CreateCreateListStep(create_list.list_expr())); - program_builder.AddStep( - ExpressionStep::MakeGenericStep(std::move(step), create_list.id())); + program_builder.AddStep(CreateCreateListStep( + create_list.list_expr().elements().size(), {}, create_list.id())); program_builder.ExitSubexpression(&create_list); std::shared_ptr arena; @@ -376,10 +374,8 @@ TEST_F(UpdatedConstantFoldingTest, CreatesLargeList) { program_builder.ExitSubexpression(&elem4); // createlist - ASSERT_OK_AND_ASSIGN(auto step_large, - CreateCreateListStep(create_list.list_expr())); - program_builder.AddStep( - ExpressionStep::MakeGenericStep(std::move(step_large), create_list.id())); + program_builder.AddStep(CreateCreateListStep( + create_list.list_expr().elements().size(), {}, create_list.id())); program_builder.ExitSubexpression(&create_list); std::shared_ptr arena; diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index fa0d556b6..5677f2a98 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -507,6 +507,17 @@ absl::flat_hash_set MakeOptionalIndicesSet( return optional_indices; } +absl::flat_hash_set MakeOptionalIndicesSet( + const cel::ListExpr& list_expr) { + absl::flat_hash_set optional_indices; + for (size_t i = 0; i < list_expr.elements().size(); ++i) { + if (list_expr.elements()[i].optional()) { + optional_indices.insert(i); + } + } + return optional_indices; +} + class FlatExprVisitor : public cel::AstVisitor { public: enum class CallHandlerResult { @@ -1350,7 +1361,17 @@ class FlatExprVisitor : public cel::AstVisitor { } } } - AddStep(CreateCreateListStep(list_expr), expr.id()); + absl::flat_hash_set optional_indices = + MakeOptionalIndicesSet(list_expr); + for (size_t index : optional_indices) { + if (!ValidateOrError(index < list_expr.elements().size(), + "Optional index out of range: ", index, + ", list size: ", list_expr.elements().size())) { + return; + } + } + AddStep(CreateCreateListStep(list_expr.elements().size(), + std::move(optional_indices), expr.id())); } // CreateStruct node handler. diff --git a/eval/eval/BUILD b/eval/eval/BUILD index f505954bf..55b1e5e29 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -36,6 +36,7 @@ cc_library( name = "evaluator_core", srcs = [ "comprehension_step.cc", + "create_list_step.cc", "equality_steps.cc", "evaluator_core.cc", "function_step.cc", @@ -45,6 +46,7 @@ cc_library( ], hdrs = [ "comprehension_step.h", + "create_list_step.h", "equality_steps.h", "evaluator_core.h", "function_step.h", @@ -65,6 +67,7 @@ cc_library( "//common:expr", "//common:function_descriptor", "//common:kind", + "//common:optional_ref", "//common:value", "//common:value_kind", "//eval/internal:errors", @@ -83,6 +86,7 @@ cc_library( "@com_google_absl//absl/base", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", @@ -302,29 +306,6 @@ cc_library( ], ) -cc_library( - name = "create_list_step", - srcs = [ - "create_list_step.cc", - ], - hdrs = [ - "create_list_step.h", - ], - deps = [ - ":attribute_utility", - ":evaluator_core", - ":expression_step_base", - ":expression_step_logic", - "//common:expr", - "//common:value", - "//internal:status_macros", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/types:optional", - ], -) - cc_library( name = "create_struct_step", srcs = [ @@ -658,7 +639,6 @@ cc_test( ], deps = [ ":cel_expression_flat_impl", - ":create_list_step", ":evaluator_core", "//base:attributes", "//base:data", diff --git a/eval/eval/create_list_step.cc b/eval/eval/create_list_step.cc index 20a9b4f37..d812486f2 100644 --- a/eval/eval/create_list_step.cc +++ b/eval/eval/create_list_step.cc @@ -4,20 +4,17 @@ #include #include #include -#include #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "absl/types/optional.h" -#include "common/expr.h" +#include "absl/types/span.h" +#include "common/optional_ref.h" #include "common/value.h" #include "common/values/list_value_builder.h" #include "eval/eval/attribute_utility.h" #include "eval/eval/evaluator_core.h" -#include "eval/eval/expression_step_base.h" -#include "eval/eval/expression_step_logic.h" -#include "internal/status_macros.h" namespace google::api::expr::runtime { @@ -29,69 +26,72 @@ using ::cel::UnknownValue; using ::cel::Value; using ::cel::common_internal::NewListValueBuilder; -class CreateListStep : public ExpressionStepBase { - public: - CreateListStep(int list_size, absl::flat_hash_set optional_indices) - : list_size_(list_size), optional_indices_(std::move(optional_indices)) {} +constexpr size_t kMaxSmallListSize = 58; - void Evaluate(ExecutionFrame* frame) const override; +template +struct StepTraits; - private: - absl::Status DoEvaluate(ExecutionFrame* frame, Value* result) const; +template <> +struct StepTraits { + static bool is_optional(const SmallListStepInfo& step, size_t index) { + ABSL_DCHECK_LT(index, kMaxSmallListSize); + return ((step.optional_bit_set >> index) & 1) == 1; + } - int list_size_; - absl::flat_hash_set optional_indices_; + static size_t num_elements(const SmallListStepInfo& step) { + return step.list_size; + } }; -void CreateListStep::Evaluate(ExecutionFrame* frame) const { - if (list_size_ < 0) { - frame->Abort(absl::InternalError("CreateListStep: list size is <0")); - return; +template <> +struct StepTraits { + static bool is_optional(const ListStepInfo& step, size_t index) { + return step.is_optional(index); } - if (!frame->value_stack().HasEnough(list_size_)) { - frame->Abort(absl::InternalError("CreateListStep: stack underflow")); - return; + static size_t num_elements(const ListStepInfo& step) { + return step.num_elements(); } +}; - Value result; - if (absl::Status status = DoEvaluate(frame, &result); !status.ok()) { - frame->Abort(std::move(status)); +template +void EvaluateListStepImpl(const StepInfo& step, ExecutionFrame& frame) { + const size_t num_elements = StepTraits::num_elements(step); + + if (!frame.value_stack().HasEnough(num_elements)) { + frame.Abort(absl::InternalError("CreateListStep: stack underflow")); return; } - frame->value_stack().PopAndPush(list_size_, std::move(result)); -} - -absl::Status CreateListStep::DoEvaluate(ExecutionFrame* frame, - Value* result) const { - auto args = frame->value_stack().GetSpan(list_size_); + absl::Span args = frame.value_stack().GetSpan(num_elements); - for (const auto& arg : args) { - if (arg.IsError()) { - *result = arg; - return absl::OkStatus(); + for (size_t i = 0; i < num_elements; ++i) { + if (args[i].IsError()) { + frame.value_stack().SwapAndPop(num_elements, i); + return; } } - if (frame->enable_unknowns()) { + if (frame.enable_unknowns()) { absl::optional unknown_set = - frame->attribute_utility().IdentifyAndMergeUnknowns( - args, frame->value_stack().GetAttributeSpan(list_size_), - /*use_partial=*/true, frame->arena()); + frame.attribute_utility().IdentifyAndMergeUnknowns( + args, frame.value_stack().GetAttributeSpan(num_elements), + /*use_partial=*/true, frame.arena()); if (unknown_set.has_value()) { - *result = std::move(*unknown_set); - return absl::OkStatus(); + frame.value_stack().PopAndPush(num_elements, std::move(*unknown_set)); + return; } } - ListValueBuilderPtr builder = NewListValueBuilder(frame->arena()); - builder->Reserve(args.size()); + ListValueBuilderPtr builder = NewListValueBuilder(frame.arena()); + builder->Reserve(num_elements); - for (size_t i = 0; i < args.size(); ++i) { - const auto& arg = args[i]; - if (optional_indices_.contains(static_cast(i))) { - if (auto optional_arg = arg.AsOptional(); optional_arg) { + for (size_t i = 0; i < num_elements; ++i) { + const Value& arg = args[i]; + if (StepTraits::is_optional(step, i)) { + if (cel::optional_ref optional_arg = + arg.AsOptional(); + optional_arg.has_value()) { if (!optional_arg->HasValue()) { continue; } @@ -99,42 +99,59 @@ absl::Status CreateListStep::DoEvaluate(ExecutionFrame* frame, optional_arg->Value(&optional_arg_value); if (optional_arg_value.IsError()) { // Error should never be in optional, but better safe than sorry. - *result = std::move(optional_arg_value); - return absl::OkStatus(); + frame.value_stack().PopAndPush(num_elements, + std::move(optional_arg_value)); + return; + } + if (absl::Status status = builder->Add(std::move(optional_arg_value)); + !status.ok()) { + frame.Abort(std::move(status)); + return; } - CEL_RETURN_IF_ERROR(builder->Add(std::move(optional_arg_value))); } else { - *result = cel::TypeConversionError(arg.GetTypeName(), "optional_type", - frame->arena()); - return absl::OkStatus(); + frame.value_stack().PopAndPush( + num_elements, + cel::TypeConversionError(arg.GetTypeName(), "optional_type", + frame.arena())); + return; } } else { - CEL_RETURN_IF_ERROR(builder->Add(arg)); + if (absl::Status status = builder->Add(arg); !status.ok()) { + frame.Abort(std::move(status)); + return; + } } } - *result = std::move(*builder).Build(); - return absl::OkStatus(); + frame.value_stack().PopAndPush(num_elements, std::move(*builder).Build()); } -absl::flat_hash_set MakeOptionalIndicesSet( - const cel::ListExpr& create_list_expr) { - absl::flat_hash_set optional_indices; - for (size_t i = 0; i < create_list_expr.elements().size(); ++i) { - if (create_list_expr.elements()[i].optional()) { - optional_indices.insert(static_cast(i)); - } - } - return optional_indices; +} // namespace + +void EvaluateListStep(const ListStepInfo& step, ExecutionFrame& frame) { + EvaluateListStepImpl(step, frame); } -} // namespace +void EvaluateSmallListStep(const SmallListStepInfo& step, + ExecutionFrame& frame) { + EvaluateListStepImpl(step, frame); +} -absl::StatusOr> CreateCreateListStep( - const cel::ListExpr& create_list_expr) { - return std::make_unique( - create_list_expr.elements().size(), - MakeOptionalIndicesSet(create_list_expr)); +ExpressionStep CreateCreateListStep( + size_t size, absl::flat_hash_set optional_indices, + int64_t expr_id) { + if (size <= kMaxSmallListSize) { + uint64_t optional_bit_set = 0; + for (size_t index : optional_indices) { + ABSL_DCHECK_LT(index, size); + optional_bit_set |= (uint64_t{1} << index); + } + return ExpressionStep::MakeCreateSmallListStep( + SmallListStepInfo{optional_bit_set, size}, expr_id); + } + return ExpressionStep::MakeCreateListStep( + std::make_unique(size, std::move(optional_indices)), + expr_id); } } // namespace google::api::expr::runtime diff --git a/eval/eval/create_list_step.h b/eval/eval/create_list_step.h index 9db49db32..dc8e57ed4 100644 --- a/eval/eval/create_list_step.h +++ b/eval/eval/create_list_step.h @@ -1,17 +1,49 @@ #ifndef THIRD_PARTY_CEL_CPP_EVAL_EVAL_CREATE_LIST_STEP_H_ #define THIRD_PARTY_CEL_CPP_EVAL_EVAL_CREATE_LIST_STEP_H_ -#include +#include +#include +#include -#include "absl/status/statusor.h" -#include "common/expr.h" -#include "eval/eval/expression_step_logic.h" +#include "absl/container/flat_hash_set.h" namespace google::api::expr::runtime { +class ExecutionFrame; +class ExpressionStep; + +// Small list step info is used for lists with 58 or fewer elements with 58 bits +// used as a bitset to indicate which elements are optional. +// List literals are generally small so this covers most cases. +struct SmallListStepInfo { + uint64_t optional_bit_set : 58; + size_t list_size : 6; +}; + +class ListStepInfo { + public: + explicit ListStepInfo(size_t num_elements) : num_elements_(num_elements) {} + ListStepInfo(size_t num_elements, absl::flat_hash_set opt_fields) + : num_elements_(num_elements), opt_fields_(std::move(opt_fields)) {} + + bool is_optional(size_t index) const { return opt_fields_.contains(index); } + + size_t num_elements() const { return num_elements_; } + + private: + size_t num_elements_; + absl::flat_hash_set opt_fields_; +}; + +void EvaluateListStep(const ListStepInfo& step, ExecutionFrame& frame); +void EvaluateSmallListStep(const SmallListStepInfo& step, + ExecutionFrame& frame); + // Factory method for CreateList which constructs an immutable list. -absl::StatusOr> CreateCreateListStep( - const cel::ListExpr& create_list_expr); +// The optional indices are assumed to be valid and in range [0, size). +ExpressionStep CreateCreateListStep( + size_t size, absl::flat_hash_set optional_indices = {}, + int64_t expr_id = -1); } // namespace google::api::expr::runtime diff --git a/eval/eval/create_list_step_test.cc b/eval/eval/create_list_step_test.cc index a8f02b5d8..56ac204b5 100644 --- a/eval/eval/create_list_step_test.cc +++ b/eval/eval/create_list_step_test.cc @@ -62,9 +62,7 @@ absl::StatusOr RunExpression( cel::interop_internal::CreateIntValue(value))); } - CEL_ASSIGN_OR_RETURN(auto step, CreateCreateListStep(create_list)); - path.push_back( - ExpressionStep::MakeGenericStep(std::move(step), dummy_expr.id())); + path.push_back(CreateCreateListStep(values.size(), {}, dummy_expr.id())); cel::RuntimeOptions options; if (enable_unknowns) { options.unknown_processing = cel::UnknownProcessingOptions::kAttributeOnly; @@ -101,9 +99,7 @@ absl::StatusOr RunExpressionWithCelValues( activation.InsertValue(var_name, value); } - CEL_ASSIGN_OR_RETURN(auto step0, CreateCreateListStep(create_list)); - path.push_back( - ExpressionStep::MakeGenericStep(std::move(step0), dummy_expr.id())); + path.push_back(CreateCreateListStep(values.size(), {}, dummy_expr.id())); cel::RuntimeOptions options; if (enable_unknowns) { @@ -137,9 +133,7 @@ TEST(CreateListStepTest, TestCreateListStackUnderflow) { auto& expr0 = create_list.mutable_elements().emplace_back().mutable_expr(); expr0.mutable_const_expr().set_int64_value(1); - ASSERT_OK_AND_ASSIGN(auto step0, CreateCreateListStep(create_list)); - path.push_back( - ExpressionStep::MakeGenericStep(std::move(step0), dummy_expr.id())); + path.push_back(CreateCreateListStep(1, {}, dummy_expr.id())); auto env = NewTestingRuntimeEnv(); CelExpressionFlatImpl cel_expr( diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index 07d985804..b389c38a6 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -39,6 +39,7 @@ #include "eval/eval/attribute_utility.h" #include "eval/eval/comprehension_slots.h" #include "eval/eval/comprehension_step.h" +#include "eval/eval/create_list_step.h" #include "eval/eval/equality_steps.h" #include "eval/eval/evaluator_stack.h" #include "eval/eval/expression_step_logic.h" @@ -112,6 +113,9 @@ enum class ExpressionStepKind : uint16_t { // Special built-in steps for mutable lists implementing map/filter. kNewMutableList = 31, kMutableListAppend = 32, + // Create list step. + kCreateList = 33, + kCreateSmallList = 34, }; struct BoolJumpStepInfo { @@ -336,6 +340,20 @@ class ExpressionStep { return step; } + static ExpressionStep MakeCreateListStep( + std::unique_ptr step_impl, int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kCreateList, id); + step.u_.create_list_step = step_impl.release(); + return step; + } + + static ExpressionStep MakeCreateSmallListStep(SmallListStepInfo info, + int64_t id = -1) { + ExpressionStep step(ExpressionStepKind::kCreateSmallList, id); + step.u_.create_small_list_step = info; + return step; + } + private: static ABSL_ATTRIBUTE_ALWAYS_INLINE inline void EvaluateReadSlotStep( size_t slot_index, ExecutionFrame& frame); @@ -406,6 +424,8 @@ class ExpressionStep { EagerFunctionStep* eager_function_step; LazyFunctionStep* lazy_function_step; std::string* identifier; + ListStepInfo* create_list_step; + SmallListStepInfo create_small_list_step; Data() : empty(nullptr) {} ~Data() {} @@ -982,6 +1002,9 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kLazyFunction: delete u_.lazy_function_step; break; + case ExpressionStepKind::kCreateList: + delete u_.create_list_step; + break; case ExpressionStepKind::kMovedFrom: case ExpressionStepKind::kIntConstant: case ExpressionStepKind::kBoolConstant: @@ -1010,6 +1033,7 @@ inline ExpressionStep::~ExpressionStep() { case ExpressionStepKind::kFastNotEqual: case ExpressionStepKind::kNewMutableList: case ExpressionStepKind::kMutableListAppend: + case ExpressionStepKind::kCreateSmallList: break; default: ABSL_UNREACHABLE(); @@ -1202,6 +1226,12 @@ inline void ExpressionStep::Evaluate(ExecutionFrame& frame) const { case ExpressionStepKind::kMutableListAppend: EvaluateMutableListAppendStep(frame); break; + case ExpressionStepKind::kCreateList: + EvaluateListStep(*u_.create_list_step, frame); + break; + case ExpressionStepKind::kCreateSmallList: + EvaluateSmallListStep(u_.create_small_list_step, frame); + break; case ExpressionStepKind::kMovedFrom: frame.Abort(absl::InternalError( "ExpressionStep::Evaluate called on moved-from step")); From 21136638a5543b832a67083bbca6b8e0cf14b6a3 Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 13 Oct 2026 15:34:27 -0700 Subject: [PATCH 297/350] Reject unexpected receiver targets in comprehension list append and map insert optimizations. PiperOrigin-RevId: 998934280 --- eval/compiler/flat_expr_builder.cc | 6 ++-- eval/compiler/flat_expr_builder_test.cc | 44 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 5677f2a98..660e34182 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -343,7 +343,8 @@ bool IsOptimizableListAppend(const cel::ComprehensionExpr* comprehension, call_expr = &(call_expr->args()[1].call_expr()); } - return call_expr->function() == cel::builtin::kAdd && + return !call_expr->has_target() && + call_expr->function() == cel::builtin::kAdd && call_expr->args().size() == 2 && call_expr->args()[0].has_ident_expr() && call_expr->args()[0].ident_expr().name() == accu_var && @@ -416,7 +417,8 @@ bool IsOptimizableMapInsert(const cel::ComprehensionExpr* comprehension, } call_expr = &(call_expr->args()[1].call_expr()); } - return call_expr->function() == "cel.@mapInsert" && + return !call_expr->has_target() && + call_expr->function() == "cel.@mapInsert" && (call_expr->args().size() == 2 || call_expr->args().size() == 3) && call_expr->args()[0].has_ident_expr() && call_expr->args()[0].ident_expr().name() == accu_var; diff --git a/eval/compiler/flat_expr_builder_test.cc b/eval/compiler/flat_expr_builder_test.cc index f9d302051..5ce27a603 100644 --- a/eval/compiler/flat_expr_builder_test.cc +++ b/eval/compiler/flat_expr_builder_test.cc @@ -3104,6 +3104,50 @@ TEST(FlatExprBuilderTest, EXPECT_THAT(result, test::IsCelBool(true)); } +TEST(FlatExprBuilderTest, ListAppendComprehensionWithTargetFailsPlanning) { + ParsedExpr parsed_expr; + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString( + R"pb( + expr { + comprehension_expr { + iter_var: "x" + iter_range { + list_expr { + elements { const_expr { int64_value: 1 } } + elements { const_expr { int64_value: 2 } } + elements { const_expr { int64_value: 3 } } + elements { const_expr { int64_value: 4 } } + } + } + accu_var: "__result__" + accu_init { list_expr {} } + loop_condition { const_expr { bool_value: true } } + loop_step { + call_expr { + target { const_expr { int64_value: 0 } } + function: "_+_" + args { ident_expr { name: "__result__" } } + args { + list_expr { elements { const_expr { int64_value: 1 } } } + } + } + } + result { ident_expr { name: "__result__" } } + } + } + )pb", + &parsed_expr)); + + cel::RuntimeOptions options; + options.enable_comprehension_list_append = true; + CelExpressionBuilderFlatImpl builder(NewTestingRuntimeEnv(), options); + ASSERT_THAT(RegisterBuiltinFunctions(builder.GetRegistry()), IsOk()); + + EXPECT_THAT( + builder.CreateExpression(&parsed_expr.expr(), &parsed_expr.source_info()), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + } // namespace } // namespace google::api::expr::runtime From 234f0b83bae708b64b1e70f3d5a9c2a2fd55282e Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Tue, 13 Oct 2026 16:48:03 -0700 Subject: [PATCH 298/350] Track iterator stack limit separately from comprehension slots. PiperOrigin-RevId: 998976105 --- eval/compiler/constant_folding.cc | 5 ++- eval/compiler/flat_expr_builder.cc | 52 ++---------------------------- eval/eval/BUILD | 1 - eval/eval/equality_steps_test.cc | 8 ++--- eval/eval/evaluator_core.cc | 8 ++--- eval/eval/evaluator_core.h | 21 ++++++------ eval/eval/evaluator_core_test.cc | 4 +-- eval/eval/iterator_stack.h | 3 +- eval/eval/lazy_init_step_test.cc | 3 +- 9 files changed, 28 insertions(+), 77 deletions(-) diff --git a/eval/compiler/constant_folding.cc b/eval/compiler/constant_folding.cc index 4a8d55c31..fb58d7d48 100644 --- a/eval/compiler/constant_folding.cc +++ b/eval/compiler/constant_folding.cc @@ -75,9 +75,8 @@ class ConstantFoldingExtension : public ProgramOptimizer { const TypeProvider& type_provider) : shared_arena_(std::move(shared_arena)), shared_message_factory_(std::move(shared_message_factory)), - state_(kDefaultStackLimit, kComprehensionSlotCount, - /*iterator_stack_size=*/0, type_provider, descriptor_pool, - message_factory, arena) {} + state_(kDefaultStackLimit, kComprehensionSlotCount, type_provider, + descriptor_pool, message_factory, arena) {} absl::Status OnPreVisit(google::api::expr::runtime::PlannerContext& context, const Expr& node) override; diff --git a/eval/compiler/flat_expr_builder.cc b/eval/compiler/flat_expr_builder.cc index 660e34182..f0920bf18 100644 --- a/eval/compiler/flat_expr_builder.cc +++ b/eval/compiler/flat_expr_builder.cc @@ -105,12 +105,7 @@ class FlatExprVisitor; // Helper for bookkeeping variables mapped to indexes. class IndexManager { public: - IndexManager() - : next_free_slot_(0), - max_slot_count_(0), - iter_stack_count_(0), - max_iter_stack_count_(0), - lazy_iter_stack_count_(0) {} + IndexManager() : next_free_slot_(0), max_slot_count_(0) {} size_t ReserveSlots(size_t n) { size_t result = next_free_slot_; @@ -126,30 +121,11 @@ class IndexManager { return next_free_slot_; } - void ReserveIterStack() { - ++iter_stack_count_; - max_iter_stack_count_ = std::max(max_iter_stack_count_, iter_stack_count_); - } - void ReleaseIterStack() { --iter_stack_count_; } - void AddLazyIterStack() { ++lazy_iter_stack_count_; } - size_t max_slot_count() const { return max_slot_count_; } - // Returns the maximum number of iterator stack slots required to evaluate the - // expression by tracking the maximum comprehension nesting in the main - // expression and dedicating a slot for any comprehensions evaluated lazily. - // This is not as tight as possible, but is a correct upper bound estimate and - // is tight for small expressions. - size_t IterStackRequirement() const { - return max_iter_stack_count_ + lazy_iter_stack_count_; - } - private: size_t next_free_slot_; size_t max_slot_count_; - size_t iter_stack_count_; - size_t max_iter_stack_count_; - size_t lazy_iter_stack_count_; }; // Helper for computing jump offsets. @@ -1169,34 +1145,21 @@ class FlatExprVisitor : public cel::AstVisitor { slot_count = 0; } } - // If this is in the scope of an optimized bind accu-init, account the slots // to the outermost bind-init scope. // // The init expression is effectively inlined at the first usage in the // critical path (which is unknown at plan time), so the used slots need to // be dedicated for the entire scope of that bind. - bool in_bind_init = false; for (ComprehensionStackRecord& record : comprehension_stack_) { if (record.in_accu_init && record.is_optimizable_bind) { record.slot_count += slot_count; slot_count = 0; - in_bind_init = true; break; } // If no bind init subexpression, account normally. } - const bool is_stack_releasable = - !in_bind_init && (!block_.has_value() || !block_->in); - if (!is_bind) { - if (is_stack_releasable) { - index_manager_.AddLazyIterStack(); - } else { - index_manager_.ReserveIterStack(); - } - } - comprehension_stack_.push_back( {&expr, &comprehension, iter_slot, iter2_slot, accu_slot, slot_count, /*subexpression=*/-1, @@ -1209,7 +1172,6 @@ class FlatExprVisitor : public cel::AstVisitor { options_.enable_comprehension_mutable_map, options_.short_circuiting), /*.is_optimizable_bind=*/is_bind, - /*.is_stack_releasable=*/is_stack_releasable, /*.iter_var_in_scope=*/false, /*.iter_var2_in_scope=*/false, /*.accu_var_in_scope=*/false, @@ -1236,9 +1198,6 @@ class FlatExprVisitor : public cel::AstVisitor { record.visitor->PostVisit(&expr); index_manager_.ReleaseSlots(record.slot_count); - if (record.is_stack_releasable) { - index_manager_.ReleaseIterStack(); - } comprehension_stack_.pop_back(); } @@ -1540,10 +1499,6 @@ class FlatExprVisitor : public cel::AstVisitor { size_t slot_count() const { return index_manager_.max_slot_count(); } - size_t iter_stack_count() const { - return index_manager_.IterStackRequirement(); - } - void AddOptimizer(std::unique_ptr optimizer) { program_optimizers_.push_back(std::move(optimizer)); } @@ -1575,7 +1530,6 @@ class FlatExprVisitor : public cel::AstVisitor { bool is_optimizable_list_append; bool is_optimizable_map_insert; bool is_optimizable_bind; - bool is_stack_releasable; bool iter_var_in_scope; bool iter_var2_in_scope; bool accu_var_in_scope; @@ -2344,8 +2298,8 @@ absl::StatusOr FlatExprBuilder::CreateExpressionImpl( FlattenExpressionTable(program_builder, execution_path); return FlatExpression(std::move(execution_path), std::move(subexpressions), - visitor.slot_count(), visitor.iter_stack_count(), - GetTypeProvider(), options_, std::move(arena)); + visitor.slot_count(), GetTypeProvider(), options_, + std::move(arena)); } const cel::TypeProvider& FlatExprBuilder::GetTypeProvider() const { return use_legacy_type_provider_ diff --git a/eval/eval/BUILD b/eval/eval/BUILD index 55b1e5e29..fa6c95a9b 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -929,7 +929,6 @@ cc_library( deps = [ "//common:value", "@com_google_absl//absl/base:nullability", - "@com_google_absl//absl/container:inlined_vector", "@com_google_absl//absl/log:absl_check", ], ) diff --git a/eval/eval/equality_steps_test.cc b/eval/eval/equality_steps_test.cc index 50970ca44..4ebfe10ef 100644 --- a/eval/eval/equality_steps_test.cc +++ b/eval/eval/equality_steps_test.cc @@ -81,7 +81,7 @@ TEST(IterativeTest, PartialAttrUnknown) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + /*comprehension_slot_count=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); @@ -113,7 +113,7 @@ TEST(IterativeTest, PartialAttrUnknownDisabled) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + /*comprehension_slot_count=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); @@ -183,7 +183,7 @@ TEST_P(EqualsTest, Iterative) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + /*comprehension_slot_count=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); @@ -328,7 +328,7 @@ TEST_P(InTest, Iterative) { FlatExpressionEvaluatorState state( /*value_stack_size=*/5, - /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + /*comprehension_slot_count=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); diff --git a/eval/eval/evaluator_core.cc b/eval/eval/evaluator_core.cc index 2fc6eda93..cbc47d136 100644 --- a/eval/eval/evaluator_core.cc +++ b/eval/eval/evaluator_core.cc @@ -222,16 +222,16 @@ FlatExpressionEvaluatorState FlatExpression::MakeEvaluatorState( google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) const { return FlatExpressionEvaluatorState(path_.size(), comprehension_slots_size_, - iterator_stack_size_, type_provider_, - descriptor_pool, message_factory, arena); + type_provider_, descriptor_pool, + message_factory, arena); } FlatExpressionEvaluatorState FlatExpression::MakeEvaluatorState( const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory) const { return FlatExpressionEvaluatorState(path_.size(), comprehension_slots_size_, - iterator_stack_size_, type_provider_, - descriptor_pool, message_factory); + type_provider_, descriptor_pool, + message_factory); } absl::StatusOr FlatExpression::EvaluateWithCallback( diff --git a/eval/eval/evaluator_core.h b/eval/eval/evaluator_core.h index b389c38a6..e415487c3 100644 --- a/eval/eval/evaluator_core.h +++ b/eval/eval/evaluator_core.h @@ -450,11 +450,14 @@ class FlatExpressionEvaluatorState { public: FlatExpressionEvaluatorState( size_t value_stack_size, size_t comprehension_slot_count, - size_t iterator_stack_size, const cel::TypeProvider& type_provider, + const cel::TypeProvider& type_provider, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nullable message_factory = nullptr) : value_stack_(value_stack_size), - iterator_stack_(iterator_stack_size), + // We currently use comprehension_slot_count because it is less of an + // over estimate than value_stack_size. In future we should just + // calculate the correct capacity. + iterator_stack_(comprehension_slot_count), comprehension_slots_(comprehension_slot_count), type_provider_(type_provider), descriptor_pool_(descriptor_pool), @@ -462,12 +465,15 @@ class FlatExpressionEvaluatorState { FlatExpressionEvaluatorState( size_t value_stack_size, size_t comprehension_slot_count, - size_t iterator_stack_size, const cel::TypeProvider& type_provider, + const cel::TypeProvider& type_provider, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, google::protobuf::Arena* absl_nonnull arena) : value_stack_(value_stack_size), - iterator_stack_(iterator_stack_size), + // We currently use comprehension_slot_count because it is less of an + // over estimate than value_stack_size. In future we should just + // calculate the correct capacity. + iterator_stack_(comprehension_slot_count), comprehension_slots_(comprehension_slot_count), type_provider_(type_provider), descriptor_pool_(descriptor_pool), @@ -873,21 +879,19 @@ class FlatExpression { : path_(std::move(path)), subexpressions_({path_}), comprehension_slots_size_(comprehension_slots_size), - iterator_stack_size_(comprehension_slots_size), type_provider_(type_provider), options_(options), arena_(std::move(arena)) {} FlatExpression(ExecutionPath path, std::vector subexpressions, - size_t comprehension_slots_size, size_t iterator_stack_size, + size_t comprehension_slots_size, const cel::TypeProvider& type_provider, const cel::RuntimeOptions& options, absl_nullable std::shared_ptr arena = nullptr) : path_(std::move(path)), subexpressions_(std::move(subexpressions)), comprehension_slots_size_(comprehension_slots_size), - iterator_stack_size_(iterator_stack_size), type_provider_(type_provider), options_(options), arena_(std::move(arena)) {} @@ -930,15 +934,12 @@ class FlatExpression { size_t comprehension_slots_size() const { return comprehension_slots_size_; } - size_t iterator_stack_size() const { return iterator_stack_size_; } - const cel::TypeProvider& type_provider() const { return type_provider_; } private: ExecutionPath path_; std::vector subexpressions_; size_t comprehension_slots_size_; - size_t iterator_stack_size_; const cel::TypeProvider& type_provider_; cel::RuntimeOptions options_; // Arena used during planning phase, may hold constant values so should be diff --git a/eval/eval/evaluator_core_test.cc b/eval/eval/evaluator_core_test.cc index 1f43f3f35..c67dc8b58 100644 --- a/eval/eval/evaluator_core_test.cc +++ b/eval/eval/evaluator_core_test.cc @@ -84,7 +84,7 @@ TEST(EvaluatorCoreTest, ExecutionFrameNext) { cel::Activation activation; FlatExpressionEvaluatorState state( path.size(), - /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + /*comprehension_slots_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); ExecutionFrame frame(path, activation, options, state); @@ -111,7 +111,7 @@ TEST(EvaluatorCoreTest, JumpOutOfRangeAborts) { cel::Activation activation; FlatExpressionEvaluatorState state( path.size(), - /*comprehension_slot_count=*/0, /*iterator_stack_size=*/0, type_provider, + /*comprehension_slots_size=*/0, type_provider, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena); ExecutionFrame frame(path, activation, options, state); diff --git a/eval/eval/iterator_stack.h b/eval/eval/iterator_stack.h index ccb377dd3..0b0de14aa 100644 --- a/eval/eval/iterator_stack.h +++ b/eval/eval/iterator_stack.h @@ -20,7 +20,6 @@ #include #include "absl/base/nullability.h" -#include "absl/container/inlined_vector.h" #include "absl/log/absl_check.h" #include "common/value.h" @@ -93,7 +92,7 @@ class IteratorStack final { } private: - absl::InlinedVector entries_; + std::vector entries_; size_t max_size_; }; diff --git a/eval/eval/lazy_init_step_test.cc b/eval/eval/lazy_init_step_test.cc index d7211e597..3b5282d21 100644 --- a/eval/eval/lazy_init_step_test.cc +++ b/eval/eval/lazy_init_step_test.cc @@ -46,8 +46,7 @@ class LazyInitStepTest : public testing::Test { public: LazyInitStepTest() : type_provider_(cel::internal::GetTestingDescriptorPool()), - evaluator_state_(kValueStack, kComprehensionSlotCount, - /*iterator_stack_size=*/0, type_provider_, + evaluator_state_(kValueStack, kComprehensionSlotCount, type_provider_, cel::internal::GetTestingDescriptorPool(), cel::internal::GetTestingMessageFactory(), &arena_) {} From cdad3a11d55a73377501b6cc7a7ee87d246a93ba Mon Sep 17 00:00:00 2001 From: Justin King Date: Wed, 14 Oct 2026 13:39:43 -0700 Subject: [PATCH 299/350] Allow updating `cel::UnknownValue` in-place PiperOrigin-RevId: 999588834 --- common/internal/unknowns.cc | 10 ++++------ common/internal/unknowns.h | 37 ++++++++++++++++++++++++++++++------- common/value.h | 13 +++++++++---- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/common/internal/unknowns.cc b/common/internal/unknowns.cc index 34fc05ed6..9f9061585 100644 --- a/common/internal/unknowns.cc +++ b/common/internal/unknowns.cc @@ -159,7 +159,7 @@ std::string UnknownTree::ToString( return result; } -UnknownSet UnknownTree::GraftImpl(const UnknownSet& in) { +void UnknownTree::GraftImpl(const UnknownSet& in, UnknownSet& out) { ABSL_DCHECK_NE(in.root, RootIfPresent()); // This should only happen in rare circumstances where somebody passed an // UnknownValue from one expression evaluation into another. @@ -184,11 +184,9 @@ UnknownSet UnknownTree::GraftImpl(const UnknownSet& in) { std::sort(unknown_function_set->nodes, unknown_function_set->nodes + unknown_function_set->size); } - return UnknownSet{ - .root = Root(), - .attributes = unknown_attribute_set, - .functions = unknown_function_set, - }; + out.root = Root(); + out.attributes = unknown_attribute_set; + out.functions = unknown_function_set; } UnknownAttributeNode* UnknownTree::GraftImpl(const UnknownAttributeNode* in) { diff --git a/common/internal/unknowns.h b/common/internal/unknowns.h index d23517265..b0bcc7dba 100644 --- a/common/internal/unknowns.h +++ b/common/internal/unknowns.h @@ -22,6 +22,7 @@ #include #include +#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/container/flat_hash_set.h" @@ -258,7 +259,9 @@ class UnknownTree { UnknownTree(const UnknownTree&) = delete; UnknownTree& operator=(const UnknownTree&) = delete; - explicit UnknownTree(google::protobuf::Arena* absl_nonnull arena) : arena_(arena) {} + explicit UnknownTree(google::protobuf::Arena* absl_nonnull arena) : arena_(arena) { + ABSL_DCHECK(arena != nullptr); + } void MaybeReset(google::protobuf::Arena* absl_nonnull arena) { ABSL_DCHECK(arena != nullptr); @@ -345,21 +348,41 @@ class UnknownTree { // address for efficiency. This function does that grafting when it is // required. [[nodiscard]] - UnknownSet Graft(const UnknownSet& in) { + ABSL_ATTRIBUTE_ALWAYS_INLINE UnknownSet Graft(const UnknownSet& in) { // Fast path, should be taken almost always. + UnknownSet out; + if (ABSL_PREDICT_TRUE(in.root == nullptr || in.root == RootIfPresent())) { + ABSL_DCHECK(in.root != nullptr || in.attributes == nullptr); + ABSL_DCHECK(in.root != nullptr || in.functions == nullptr); + out = in; + } else { + // Slow path, should be taken extremely rarely. + GraftImpl(in, out); + } + return out; + } + + // In the rare event that somebody manages to pass an unknown set from one + // expression into another, we need to be able to graft those nodes into our + // tree. The unknown attribute set and unknown function set sort nodes by + // address for efficiency. This function does that grafting when it is + // required. + ABSL_ATTRIBUTE_ALWAYS_INLINE void GraftAt(UnknownSet& in) { if (ABSL_PREDICT_TRUE(in.root == nullptr || in.root == RootIfPresent())) { - return in; + // Fast path, should be taken almost always. + ABSL_DCHECK(in.root != nullptr || in.attributes == nullptr); + ABSL_DCHECK(in.root != nullptr || in.functions == nullptr); + } else { + // Slow path, should be taken extremely rarely. + GraftImpl(in, in); } - // Slow path, should be taken extremely rarely. - return GraftImpl(in); } private: friend class UnknownAttributeNode; friend class UnknownRoot; - [[nodiscard]] - UnknownSet GraftImpl(const UnknownSet& in); + ABSL_ATTRIBUTE_NOINLINE void GraftImpl(const UnknownSet& in, UnknownSet& out); [[nodiscard]] UnknownAttributeNode* GraftImpl(const UnknownAttributeNode* in); diff --git a/common/value.h b/common/value.h index b8801c00e..fc7992c49 100644 --- a/common/value.h +++ b/common/value.h @@ -1020,9 +1020,7 @@ class Value final : private common_internal::ValueMixin { // Performs a checked cast from a value to an unknown value, // returning a non-empty optional with either a value or reference to the // unknown value. Otherwise an empty optional is returned. - optional_ref AsUnknown() & ABSL_ATTRIBUTE_LIFETIME_BOUND { - return std::as_const(*this).AsUnknown(); - } + optional_ref AsUnknown() & ABSL_ATTRIBUTE_LIFETIME_BOUND; optional_ref AsUnknown() const& ABSL_ATTRIBUTE_LIFETIME_BOUND; absl::optional AsUnknown() &&; @@ -1666,7 +1664,7 @@ class Value final : private common_internal::ValueMixin { // `AsUnknown()`. template std::enable_if_t, - optional_ref> + optional_ref> As() & ABSL_ATTRIBUTE_LIFETIME_BOUND { return AsUnknown(); } @@ -3193,6 +3191,13 @@ inline optional_ref Value::AsUnknown() const& { return std::nullopt; } +inline optional_ref Value::AsUnknown() & { + if (auto* alternative = variant_.As(); alternative != nullptr) { + return *alternative; + } + return std::nullopt; +} + inline absl::optional Value::AsUnknown() && { if (auto* alternative = variant_.As(); alternative != nullptr) { return std::move(*alternative); From 268742fbd6314ba849b75edfb4df343518de0a8e Mon Sep 17 00:00:00 2001 From: Justin King Date: Wed, 14 Oct 2026 14:10:29 -0700 Subject: [PATCH 300/350] Implement three-way comparison for `cel::Attribute` and friends PiperOrigin-RevId: 999611174 --- base/BUILD | 1 + base/attribute.cc | 430 ++++++++++++++++++++++++++++------------------ base/attribute.h | 167 +++++++++++++++--- 3 files changed, 411 insertions(+), 187 deletions(-) diff --git a/base/BUILD b/base/BUILD index cf116727d..9becb06cb 100644 --- a/base/BUILD +++ b/base/BUILD @@ -47,6 +47,7 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:compare", "@com_google_absl//absl/types:optional_ref", "@com_google_absl//absl/types:span", ], diff --git a/base/attribute.cc b/base/attribute.cc index 0375067d8..2d8d8cada 100644 --- a/base/attribute.cc +++ b/base/attribute.cc @@ -14,6 +14,7 @@ #include "base/attribute.h" +#include #include #include #include @@ -21,7 +22,7 @@ #include #include "absl/algorithm/container.h" -#include "absl/base/macros.h" +#include "absl/base/attributes.h" #include "absl/base/nullability.h" #include "absl/log/absl_check.h" #include "absl/meta/type_traits.h" @@ -29,6 +30,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "absl/types/compare.h" #include "base/kind.h" #include "internal/lexis.h" #include "internal/status_macros.h" @@ -195,58 +197,6 @@ Kind AttributeQualifier::kind() const { return std::visit(AttributeQualifierTypeVisitor{}, value_); } -bool Attribute::operator==(const Attribute& other) const { - // We cannot check pointer equality as a short circuit because we have to - // treat all invalid AttributeQualifier as not equal to each other. - // TODO(issues/41) we only support Ident-rooted attributes at the moment. - if (variable_name() != other.variable_name()) { - return false; - } - - if (qualifier_path().size() != other.qualifier_path().size()) { - return false; - } - - for (size_t i = 0; i < qualifier_path().size(); i++) { - if (!(qualifier_path()[i] == other.qualifier_path()[i])) { - return false; - } - } - - return true; -} - -bool Attribute::operator<(const Attribute& other) const { - if (impl_.get() == other.impl_.get()) { - return false; - } - auto lhs_begin = qualifier_path().begin(); - auto lhs_end = qualifier_path().end(); - auto rhs_begin = other.qualifier_path().begin(); - auto rhs_end = other.qualifier_path().end(); - while (lhs_begin != lhs_end && rhs_begin != rhs_end) { - if (*lhs_begin < *rhs_begin) { - return true; - } - if (!(*lhs_begin == *rhs_begin)) { - return false; - } - lhs_begin++; - rhs_begin++; - } - if (lhs_begin == lhs_end && rhs_begin == rhs_end) { - // Neither has any elements left, they are equal. Compare variable names. - return variable_name() < other.variable_name(); - } - if (lhs_begin == lhs_end) { - // Left has no more elements. Right is greater. - return true; - } - // Right has no more elements. Left is greater. - ABSL_ASSERT(rhs_begin == rhs_end); - return false; -} - absl::StatusOr Attribute::AsString() const { if (variable_name().empty()) { return absl::InvalidArgumentError( @@ -368,65 +318,103 @@ bool AttributeQualifierPattern::IsMatch(absl::string_view other_key) const { namespace { struct AttributeQualifierEqualTo { - bool operator()(const std::monostate&, const std::monostate&) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(const std::monostate&, + const std::monostate&) const { return false; } template - std::enable_if_t, std::monostate>, - bool> + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< + !std::is_same_v, std::monostate>, bool> operator()(const std::monostate&, const T&) const { return false; } template - std::enable_if_t, std::monostate>, - bool> + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< + !std::is_same_v, std::monostate>, bool> operator()(const T&, const std::monostate&) const { return false; } - bool operator()(bool lhs, bool rhs) const { return lhs == rhs; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(bool lhs, bool rhs) const { + return lhs == rhs; + } - bool operator()(bool, int64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(bool, int64_t) const { + return false; + } - bool operator()(bool, uint64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(bool, uint64_t) const { + return false; + } - bool operator()(bool, absl::string_view) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(bool, absl::string_view) const { + return false; + } - bool operator()(int64_t, bool) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(int64_t, bool) const { + return false; + } - bool operator()(int64_t lhs, int64_t rhs) const { return lhs == rhs; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(int64_t lhs, int64_t rhs) const { + return lhs == rhs; + } - bool operator()(int64_t, uint64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(int64_t, uint64_t) const { + return false; + } - bool operator()(int64_t, absl::string_view) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(int64_t, + absl::string_view) const { + return false; + } - bool operator()(uint64_t, bool) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(uint64_t, bool) const { + return false; + } - bool operator()(uint64_t, int64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(uint64_t, int64_t) const { + return false; + } - bool operator()(uint64_t lhs, uint64_t rhs) const { return lhs == rhs; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(uint64_t lhs, + uint64_t rhs) const { + return lhs == rhs; + } - bool operator()(uint64_t, absl::string_view) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(uint64_t, + absl::string_view) const { + return false; + } - bool operator()(absl::string_view, bool) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(absl::string_view, bool) const { + return false; + } - bool operator()(absl::string_view, int64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(absl::string_view, + int64_t) const { + return false; + } - bool operator()(absl::string_view, uint64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(absl::string_view, + uint64_t) const { + return false; + } - bool operator()(absl::string_view lhs, absl::string_view rhs) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()(absl::string_view lhs, + absl::string_view rhs) const { return lhs == rhs; } - bool operator()(const common_internal::WildcardType&, - const common_internal::WildcardType&) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool operator()( + const common_internal::WildcardType&, + const common_internal::WildcardType&) const { return true; } template - std::enable_if_t< + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< !std::is_same_v, common_internal::WildcardType> && !std::is_same_v, std::monostate>, bool> @@ -435,7 +423,7 @@ struct AttributeQualifierEqualTo { } template - std::enable_if_t< + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< !std::is_same_v, common_internal::WildcardType> && !std::is_same_v, std::monostate>, bool> @@ -444,85 +432,243 @@ struct AttributeQualifierEqualTo { } }; -struct AttributeQualifierLess { - bool operator()(const std::monostate&, const std::monostate&) const { - return false; +template +[[nodiscard]] +ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering SynthesizeCompare(T lhs, + T rhs) { +#if defined(__cpp_impl_three_way_comparison) && \ + defined(__cpp_lib_three_way_comparison) + return lhs <=> rhs; +#else + if (lhs < rhs) { + return absl::strong_ordering::less; + } + if (rhs < lhs) { + return absl::strong_ordering::greater; + } + return absl::strong_ordering::equal; +#endif +} + +struct AttributeQualifierCompare { + // monostate < int < uint < string < bool < wildcard + + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + const std::monostate&, const std::monostate&) const { + return absl::strong_ordering::equal; } template - std::enable_if_t, std::monostate>, - bool> - operator()(const std::monostate&, const T&) const { - return true; + ABSL_ATTRIBUTE_ALWAYS_INLINE + std::enable_if_t, std::monostate>, + absl::strong_ordering> + operator()(const std::monostate&, const T&) const { + return absl::strong_ordering::less; } template - std::enable_if_t, std::monostate>, - bool> - operator()(const T&, const std::monostate&) const { - return false; + ABSL_ATTRIBUTE_ALWAYS_INLINE + std::enable_if_t, std::monostate>, + absl::strong_ordering> + operator()(const T&, const std::monostate&) const { + return absl::strong_ordering::greater; } - bool operator()(bool lhs, bool rhs) const { return lhs < rhs; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + bool lhs, bool rhs) const { + return (SynthesizeCompare)(lhs, rhs); + } - bool operator()(bool, int64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()(bool, + int64_t) const { + return absl::strong_ordering::greater; + } - bool operator()(bool, uint64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + bool, uint64_t) const { + return absl::strong_ordering::greater; + } - bool operator()(bool, absl::string_view) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + bool, absl::string_view) const { + return absl::strong_ordering::greater; + } - bool operator()(int64_t, bool) const { return true; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()(int64_t, + bool) const { + return absl::strong_ordering::less; + } - bool operator()(int64_t lhs, int64_t rhs) const { return lhs < rhs; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + int64_t lhs, int64_t rhs) const { + return (SynthesizeCompare)(lhs, rhs); + } - bool operator()(int64_t, uint64_t) const { return true; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + int64_t, uint64_t) const { + return absl::strong_ordering::less; + } - bool operator()(int64_t, absl::string_view) const { return true; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + int64_t, absl::string_view) const { + return absl::strong_ordering::less; + } - bool operator()(uint64_t, bool) const { return true; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()(uint64_t, + bool) const { + return absl::strong_ordering::less; + } - bool operator()(uint64_t, int64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()(uint64_t, + int64_t) const { + return absl::strong_ordering::greater; + } - bool operator()(uint64_t lhs, uint64_t rhs) const { return lhs < rhs; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + uint64_t lhs, uint64_t rhs) const { + return (SynthesizeCompare)(lhs, rhs); + } - bool operator()(uint64_t, absl::string_view) const { return true; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + uint64_t, absl::string_view) const { + return absl::strong_ordering::less; + } - bool operator()(absl::string_view, bool) const { return true; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + absl::string_view, bool) const { + return absl::strong_ordering::less; + } - bool operator()(absl::string_view, int64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + absl::string_view, int64_t) const { + return absl::strong_ordering::greater; + } - bool operator()(absl::string_view, uint64_t) const { return false; } + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + absl::string_view, uint64_t) const { + return absl::strong_ordering::greater; + } - bool operator()(absl::string_view lhs, absl::string_view rhs) const { - return lhs < rhs; + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + absl::string_view lhs, absl::string_view rhs) const { + return (SynthesizeCompare)(lhs, rhs); } - bool operator()(const common_internal::WildcardType&, - const common_internal::WildcardType&) const { - return false; + ABSL_ATTRIBUTE_ALWAYS_INLINE absl::strong_ordering operator()( + const common_internal::WildcardType&, + const common_internal::WildcardType&) const { + return absl::strong_ordering::equal; } template - std::enable_if_t< + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< !std::is_same_v, common_internal::WildcardType> && !std::is_same_v, std::monostate>, - bool> + absl::strong_ordering> operator()(const common_internal::WildcardType&, const T&) const { - return false; + return absl::strong_ordering::greater; } template - std::enable_if_t< + ABSL_ATTRIBUTE_ALWAYS_INLINE std::enable_if_t< !std::is_same_v, common_internal::WildcardType> && !std::is_same_v, std::monostate>, - bool> + absl::strong_ordering> operator()(const T&, const common_internal::WildcardType&) const { - return true; + return absl::strong_ordering::less; } }; } // namespace +absl::strong_ordering AttributeQualifier::Compare( + const AttributeQualifier& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifier::Compare( + const AttributeQualifierView& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifier::Compare( + const AttributeQualifierPattern& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierView::Compare( + const AttributeQualifier& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierView::Compare( + const AttributeQualifierView& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierView::Compare( + const AttributeQualifierPattern& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierPattern::Compare( + const AttributeQualifier& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierPattern::Compare( + const AttributeQualifierView& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering AttributeQualifierPattern::Compare( + const AttributeQualifierPattern& other) const { + return std::visit(AttributeQualifierCompare{}, + common_internal::AsVariant(*this), + common_internal::AsVariant(other)); +} + +absl::strong_ordering Attribute::Compare(const Attribute& other) const { + if (absl::strong_ordering order = + (SynthesizeCompare)(variable_name(), other.variable_name()); + order != absl::strong_ordering::equal) { + return order; + } + const auto& qualifiers = qualifier_path(); + const auto& other_qualifiers = other.qualifier_path(); + const size_t min_size = std::min(qualifiers.size(), other_qualifiers.size()); + for (size_t i = 0; i < min_size; ++i) { + if (absl::strong_ordering order = + qualifiers[i].Compare(other_qualifiers[i]); + order != absl::strong_ordering::equal) { + return order; + } + } + if (qualifiers.size() < other_qualifiers.size()) { + return absl::strong_ordering::less; + } + if (qualifiers.size() > other_qualifiers.size()) { + return absl::strong_ordering::greater; + } + return absl::strong_ordering::equal; +} + bool operator==(const AttributeQualifier& lhs, const AttributeQualifier& rhs) { return std::visit(AttributeQualifierEqualTo{}, common_internal::AsVariant(lhs), @@ -585,57 +731,9 @@ bool operator==(const AttributeQualifierPattern& lhs, common_internal::AsVariant(rhs)); } -bool operator<(const AttributeQualifier& lhs, const AttributeQualifier& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); -} - -bool operator<(const AttributeQualifierView& lhs, - const AttributeQualifierView& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); -} - -bool operator<(const AttributeQualifierPattern& lhs, - const AttributeQualifierPattern& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); -} - -bool operator<(const AttributeQualifier& lhs, - const AttributeQualifierView& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); -} - -bool operator<(const AttributeQualifier& lhs, - const AttributeQualifierPattern& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); -} - -bool operator<(const AttributeQualifierView& lhs, - const AttributeQualifier& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); -} - -bool operator<(const AttributeQualifierView& lhs, - const AttributeQualifierPattern& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); -} - -bool operator<(const AttributeQualifierPattern& lhs, - const AttributeQualifier& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); -} - -bool operator<(const AttributeQualifierPattern& lhs, - const AttributeQualifierView& rhs) { - return std::visit(AttributeQualifierLess{}, common_internal::AsVariant(lhs), - common_internal::AsVariant(rhs)); +bool operator==(const Attribute& lhs, const Attribute& rhs) { + return lhs.variable_name() == rhs.variable_name() && + absl::c_equal(lhs.qualifier_path(), rhs.qualifier_path()); } bool operator==(const AttributePattern& lhs, const AttributePattern& rhs) { diff --git a/base/attribute.h b/base/attribute.h index 521391c68..7e9e98c67 100644 --- a/base/attribute.h +++ b/base/attribute.h @@ -32,6 +32,7 @@ #include "absl/log/absl_check.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/compare.h" #include "absl/types/optional_ref.h" #include "absl/types/span.h" #include "base/kind.h" @@ -245,6 +246,15 @@ class AttributeQualifier { [[nodiscard]] bool IsMatch(absl::string_view other_key) const; + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifier& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierView& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierPattern& other) const; + private: friend const common_internal::AttributeQualifierVariant& common_internal::AsVariant(const AttributeQualifier& qualifier); @@ -441,6 +451,15 @@ class AttributeQualifierView { std::string* absl_nonnull out, absl::Span qualifiers); + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifier& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierView& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierPattern& other) const; + private: friend const common_internal::AttributeQualifierViewVariant& common_internal::AsVariant(const AttributeQualifierView& qualifier); @@ -729,6 +748,15 @@ class AttributeQualifierPattern { [[nodiscard]] bool IsMatch(absl::string_view other_key) const; + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifier& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierView& other) const; + + [[nodiscard]] + absl::strong_ordering Compare(const AttributeQualifierPattern& other) const; + private: friend const common_internal::AttributeQualifierPatternVariant& common_internal::AsVariant(const AttributeQualifierPattern& qualifier); @@ -782,6 +810,62 @@ bool operator==(const AttributeQualifierPattern& lhs, bool operator==(const AttributeQualifierPattern& lhs, const AttributeQualifierView& rhs); +#if defined(__cpp_impl_three_way_comparison) && \ + defined(__cpp_lib_three_way_comparison) +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifier& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs); +} + +[[nodiscard]] +inline absl::strong_ordering operator<=>(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs); +} +#else [[nodiscard]] inline bool operator!=(const AttributeQualifier& lhs, const AttributeQualifier& rhs) { @@ -837,39 +921,59 @@ inline bool operator!=(const AttributeQualifierPattern& lhs, } [[nodiscard]] -bool operator<(const AttributeQualifier& lhs, const AttributeQualifier& rhs); +inline bool operator<(const AttributeQualifier& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs) < 0; +} [[nodiscard]] -bool operator<(const AttributeQualifierView& lhs, - const AttributeQualifierView& rhs); +inline bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs) < 0; +} [[nodiscard]] -bool operator<(const AttributeQualifierPattern& lhs, - const AttributeQualifierPattern& rhs); +inline bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs) < 0; +} [[nodiscard]] -bool operator<(const AttributeQualifier& lhs, - const AttributeQualifierView& rhs); +inline bool operator<(const AttributeQualifier& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs) < 0; +} [[nodiscard]] -bool operator<(const AttributeQualifier& lhs, - const AttributeQualifierPattern& rhs); +inline bool operator<(const AttributeQualifier& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs) < 0; +} [[nodiscard]] -bool operator<(const AttributeQualifierView& lhs, - const AttributeQualifier& rhs); +inline bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs) < 0; +} [[nodiscard]] -bool operator<(const AttributeQualifierView& lhs, - const AttributeQualifierPattern& rhs); +inline bool operator<(const AttributeQualifierView& lhs, + const AttributeQualifierPattern& rhs) { + return lhs.Compare(rhs) < 0; +} [[nodiscard]] -bool operator<(const AttributeQualifierPattern& lhs, - const AttributeQualifier& rhs); +inline bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifier& rhs) { + return lhs.Compare(rhs) < 0; +} [[nodiscard]] -bool operator<(const AttributeQualifierPattern& lhs, - const AttributeQualifierView& rhs); +inline bool operator<(const AttributeQualifierPattern& lhs, + const AttributeQualifierView& rhs) { + return lhs.Compare(rhs) < 0; +} +#endif namespace common_internal { @@ -930,15 +1034,14 @@ class Attribute { return impl_->qualifier_path; } - bool operator==(const Attribute& other) const; - - bool operator<(const Attribute& other) const; - absl::StatusOr AsString() const; [[nodiscard]] std::string ToString() const; + [[nodiscard]] + absl::strong_ordering Compare(const Attribute& other) const; + private: struct Impl final { Impl(std::string variable_name, @@ -953,6 +1056,28 @@ class Attribute { std::shared_ptr impl_; }; +[[nodiscard]] +bool operator==(const Attribute& lhs, const Attribute& rhs); + +#if defined(__cpp_impl_three_way_comparison) && \ + defined(__cpp_lib_three_way_comparison) +[[nodiscard]] +inline absl::strong_ordering operator<=>(const Attribute& lhs, + const Attribute& rhs) { + return lhs.Compare(rhs); +} +#else +[[nodiscard]] +inline bool operator!=(const Attribute& lhs, const Attribute& rhs) { + return !operator==(lhs, rhs); +} + +[[nodiscard]] +inline bool operator<(const Attribute& lhs, const Attribute& rhs) { + return lhs.Compare(rhs) < 0; +} +#endif + template void AbslStringify(S& sink, const Attribute& attribute) { sink.Append(attribute.ToString()); From e026439a3652e579baddf4027213db89b23884fa Mon Sep 17 00:00:00 2001 From: Justin King Date: Wed, 14 Oct 2026 16:12:57 -0700 Subject: [PATCH 301/350] Fix bug in optimized regex matching step PiperOrigin-RevId: 999692402 --- eval/eval/regex_match_step.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eval/eval/regex_match_step.cc b/eval/eval/regex_match_step.cc index 3c949ee9f..4df758e76 100644 --- a/eval/eval/regex_match_step.cc +++ b/eval/eval/regex_match_step.cc @@ -66,6 +66,10 @@ class RegexMatchStep final : public ExpressionStepBase { } auto input_args = frame->value_stack().GetSpan(kNumRegexMatchArguments); const auto& subject = input_args[kRegexMatchStepSubject]; + if (subject.IsUnknown() || subject.IsError()) { + // Propagate unknowns and errors. + return; + } if (!subject->Is()) { frame->Abort(absl::InternalError( "First argument for regular expression match must be a string")); From bb933d0e132d16c8b9ad31dd9cc1a0f02cd6a04e Mon Sep 17 00:00:00 2001 From: Jonathan Tatum Date: Wed, 14 Oct 2026 17:05:42 -0700 Subject: [PATCH 302/350] Remove temporary enable_use_new_field_select_implementation option. PiperOrigin-RevId: 999722702 --- eval/eval/BUILD | 4 +- eval/eval/select_step.cc | 68 +++--------- eval/public/cel_options.cc | 1 - eval/public/cel_options.h | 10 -- .../structs/proto_message_type_adapter.cc | 6 -- .../structs/proto_message_type_adapter.h | 4 - extensions/BUILD | 3 - extensions/select_optimization.cc | 102 +++--------------- runtime/runtime_options.h | 10 -- 9 files changed, 32 insertions(+), 176 deletions(-) diff --git a/eval/eval/BUILD b/eval/eval/BUILD index fa6c95a9b..4bb06c7b8 100644 --- a/eval/eval/BUILD +++ b/eval/eval/BUILD @@ -287,12 +287,10 @@ cc_library( ":evaluator_core", ":expression_step_base", ":expression_step_logic", - "//common:memory", + "//common:legacy_value", "//common:type", "//common:value", "//common:value_kind", - "//eval/public:cel_value", - "//eval/public/structs:proto_message_type_adapter", "//internal:status_macros", "//runtime:runtime_options", "@com_google_absl//absl/base:nullability", diff --git a/eval/eval/select_step.cc b/eval/eval/select_step.cc index 8f58075af..297eefa2b 100644 --- a/eval/eval/select_step.cc +++ b/eval/eval/select_step.cc @@ -13,7 +13,6 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "common/legacy_value.h" -#include "common/memory.h" #include "common/type.h" #include "common/value.h" #include "common/value_kind.h" @@ -21,8 +20,6 @@ #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" #include "eval/eval/expression_step_logic.h" -#include "eval/public/cel_value.h" -#include "eval/public/structs/proto_message_type_adapter.h" #include "internal/status_macros.h" #include "runtime/runtime_options.h" #include "google/protobuf/arena.h" @@ -77,34 +74,6 @@ absl::optional CheckForMarkedAttributes(const AttributeTrail& trail, return std::nullopt; } -// Helper for StructValue::GetFieldByName. Used for opting out of old reflection -// implementation. -absl::Status WrappedStructGet( - const Value& target, absl::string_view field, - ProtoWrapperTypeOptions unboxing_option, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, - bool enable_use_new_field_select_implementation, - Value* absl_nonnull result) { - if (!enable_use_new_field_select_implementation) { - if (const google::protobuf::Message* message = - cel::interop_internal::GetLegacyMessage(target); - message != nullptr) { - CelValue::MessageWrapper message_wrapper( - message, &GetGenericProtoTypeInfoInstance()); - CEL_ASSIGN_OR_RETURN( - CelValue cel_value, - internal::GetGenericProtoAccessApisInstance().GetField( - field, message_wrapper, unboxing_option, - cel::MemoryManagerRef::Pooling(arena))); - return cel::ModernValue(arena, cel_value, *result); - } - } - return target.GetStruct().GetFieldByName( - field, unboxing_option, descriptor_pool, message_factory, arena, result); -} - absl::Status PerformHas(const Value& target, absl::string_view field, const StringValue& field_value, const google::protobuf::DescriptorPool* descriptor_pool, @@ -135,9 +104,7 @@ absl::Status PerformGet(const Value& target, absl::string_view field, ProtoWrapperTypeOptions unboxing_option, const google::protobuf::DescriptorPool* descriptor_pool, google::protobuf::MessageFactory* message_factory, - google::protobuf::Arena* arena, - bool enable_use_new_field_select_implementation, - Value& result) { + google::protobuf::Arena* arena, Value& result) { switch (target.kind()) { case ValueKind::kMap: { auto status = target.GetMap().Get(field_value, descriptor_pool, @@ -148,9 +115,9 @@ absl::Status PerformGet(const Value& target, absl::string_view field, return absl::OkStatus(); } case ValueKind::kStruct: { - auto status = WrappedStructGet( - target, field, unboxing_option, descriptor_pool, message_factory, - arena, enable_use_new_field_select_implementation, &result); + auto status = target.GetStruct().GetFieldByName( + field, unboxing_option, descriptor_pool, message_factory, arena, + &result); if (!status.ok()) { result = ErrorValue::From(std::move(status), arena); } @@ -166,9 +133,7 @@ absl::Status PerformOptionalGet(const Value& target, absl::string_view field, ProtoWrapperTypeOptions unboxing_option, const google::protobuf::DescriptorPool* descriptor_pool, google::protobuf::MessageFactory* message_factory, - google::protobuf::Arena* arena, - bool enable_use_new_field_select_implementation, - Value& result) { + google::protobuf::Arena* arena, Value& result) { switch (target.kind()) { case ValueKind::kMap: { CEL_ASSIGN_OR_RETURN( @@ -189,9 +154,9 @@ absl::Status PerformOptionalGet(const Value& target, absl::string_view field, result = OptionalValue::None(); return absl::OkStatus(); } - CEL_RETURN_IF_ERROR(WrappedStructGet( - target, field, unboxing_option, descriptor_pool, message_factory, - arena, enable_use_new_field_select_implementation, &result)); + CEL_RETURN_IF_ERROR(target.GetStruct().GetFieldByName( + field, unboxing_option, descriptor_pool, message_factory, arena, + &result)); ABSL_DCHECK(!result.IsUnknown()); result = OptionalValue::Of(std::move(result), arena); @@ -299,10 +264,10 @@ void SelectStep::Evaluate(ExecutionFrame* frame) const { } Value value; optional_arg->Value(&value); - absl::Status status = PerformOptionalGet( - value, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, - frame->descriptor_pool(), frame->message_factory(), frame->arena(), - frame->options().enable_use_new_field_select_implementation, result); + absl::Status status = + PerformOptionalGet(value, field_, cel::StringValue::WrapUnsafe(field_), + unboxing_option_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), result); if (!status.ok()) { result = ErrorValue::From(std::move(status), frame->arena()); } @@ -310,10 +275,10 @@ void SelectStep::Evaluate(ExecutionFrame* frame) const { return; } - if (absl::Status status = PerformGet( - arg, field_, cel::StringValue::WrapUnsafe(field_), unboxing_option_, - frame->descriptor_pool(), frame->message_factory(), frame->arena(), - frame->options().enable_use_new_field_select_implementation, result); + if (absl::Status status = + PerformGet(arg, field_, cel::StringValue::WrapUnsafe(field_), + unboxing_option_, frame->descriptor_pool(), + frame->message_factory(), frame->arena(), result); !status.ok()) { frame->Abort(std::move(status)); return; @@ -388,7 +353,6 @@ class ProtoSelectStep : public SelectStep { return; } else if (const google::protobuf::Message* legacy_message = cel::interop_internal::GetLegacyMessage(arg); - frame->options().enable_use_new_field_select_implementation && legacy_message != nullptr) { auto parsed_message = cel::UnsafeParsedMessageValue(legacy_message); // A little unfortunate, but need to special case for legacy values so we diff --git a/eval/public/cel_options.cc b/eval/public/cel_options.cc index 100ef5e01..93b67ad35 100644 --- a/eval/public/cel_options.cc +++ b/eval/public/cel_options.cc @@ -45,7 +45,6 @@ cel::RuntimeOptions ConvertToRuntimeOptions(const InterpreterOptions& options) { options.enable_fast_builtins, options.enable_precision_preserving_double_format, options.enable_typed_field_access, - options.enable_use_new_field_select_implementation, }; } diff --git a/eval/public/cel_options.h b/eval/public/cel_options.h index e7981a226..f28b37a3a 100644 --- a/eval/public/cel_options.h +++ b/eval/public/cel_options.h @@ -225,16 +225,6 @@ struct InterpreterOptions { // path for field access when the type is known at plan time, instead of using // the generic field access implementation. bool enable_typed_field_access = true; - - // Temporary flag to gate using a new field selection implementation for - // protos. - // - // For the cel::Runtime APIs, this is a no-op. - // - // For google::api::expr::runtime::CelExpression, this will enable updated - // implementations for field access on protobuf messages, aligned with the - // cel::Value implementation. - bool enable_use_new_field_select_implementation = true; }; // LINT.ThenChange(//depot/google3/runtime/runtime_options.h) diff --git a/eval/public/structs/proto_message_type_adapter.cc b/eval/public/structs/proto_message_type_adapter.cc index 228710c59..95127c77e 100644 --- a/eval/public/structs/proto_message_type_adapter.cc +++ b/eval/public/structs/proto_message_type_adapter.cc @@ -492,10 +492,4 @@ const LegacyTypeInfoApis& GetGenericProtoTypeInfoInstance() { return DucktypedMessageAdapter::GetSingleton(); } -namespace internal { -const LegacyTypeAccessApis& GetGenericProtoAccessApisInstance() { - return DucktypedMessageAdapter::GetSingleton(); -} -} // namespace internal - } // namespace google::api::expr::runtime diff --git a/eval/public/structs/proto_message_type_adapter.h b/eval/public/structs/proto_message_type_adapter.h index f1598abd9..aa609017b 100644 --- a/eval/public/structs/proto_message_type_adapter.h +++ b/eval/public/structs/proto_message_type_adapter.h @@ -107,10 +107,6 @@ absl::StatusOr CreateCelValueFromField( // instead of expecting a particular message type given a TypeInfo. const LegacyTypeInfoApis& GetGenericProtoTypeInfoInstance(); -namespace internal { -const LegacyTypeAccessApis& GetGenericProtoAccessApisInstance(); -} // namespace internal - } // namespace google::api::expr::runtime #endif // THIRD_PARTY_CEL_CPP_EVAL_PUBLIC_STRUCTS_PROTO_MESSAGE_TYPE_ADAPTER_H_ diff --git a/extensions/BUILD b/extensions/BUILD index 46fc088ae..93b08e6da 100644 --- a/extensions/BUILD +++ b/extensions/BUILD @@ -329,7 +329,6 @@ cc_library( "//common:expr", "//common:function_descriptor", "//common:kind", - "//common:memory", "//common:native_type", "//common:type", "//common:value", @@ -338,8 +337,6 @@ cc_library( "//eval/eval:attribute_trail", "//eval/eval:evaluator_core", "//eval/eval:expression_step_base", - "//eval/public:cel_value", - "//eval/public/structs:proto_message_type_adapter", "//internal:casts", "//internal:number", "//internal:status_macros", diff --git a/extensions/select_optimization.cc b/extensions/select_optimization.cc index dff02b625..02855dec1 100644 --- a/extensions/select_optimization.cc +++ b/extensions/select_optimization.cc @@ -43,8 +43,6 @@ #include "common/expr.h" #include "common/function_descriptor.h" #include "common/kind.h" -#include "common/legacy_value.h" -#include "common/memory.h" #include "common/native_type.h" #include "common/type.h" #include "common/value.h" @@ -53,8 +51,6 @@ #include "eval/eval/attribute_trail.h" #include "eval/eval/evaluator_core.h" #include "eval/eval/expression_step_base.h" -#include "eval/public/cel_value.h" -#include "eval/public/structs/proto_message_type_adapter.h" #include "internal/casts.h" #include "internal/number.h" #include "internal/status_macros.h" @@ -78,15 +74,12 @@ using ::cel::Expr; using ::cel::ExprKind; using ::cel::SelectExpr; using ::google::api::expr::runtime::AttributeTrail; -using ::google::api::expr::runtime::CelValue; using ::google::api::expr::runtime::ExecutionFrame; using ::google::api::expr::runtime::ExecutionFrameBase; using ::google::api::expr::runtime::ExpressionStep; using ::google::api::expr::runtime::ExpressionStepBase; -using ::google::api::expr::runtime::GetGenericProtoTypeInfoInstance; using ::google::api::expr::runtime::PlannerContext; using ::google::api::expr::runtime::ProgramOptimizer; -using ::google::api::expr::runtime::internal::GetGenericProtoAccessApisInstance; // Represents a single select operation (field access or indexing). // For struct-typed field accesses, includes the field name and the field @@ -275,69 +268,11 @@ absl::StatusOr MapKeyFromQualifier(const AttributeQualifier& qual, } } -// // Helper for StructValue::GetFieldByName. Used for opting out of old -// reflection implementation. -absl::StatusOr WrappedStructGet( - const Value& target, absl::string_view field, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, - bool enable_use_new_field_select_implementation) { - if (!enable_use_new_field_select_implementation) { - if (const google::protobuf::Message* message = - cel::interop_internal::GetLegacyMessage(target); - message != nullptr) { - CelValue::MessageWrapper message_wrapper( - message, &GetGenericProtoTypeInfoInstance()); - CEL_ASSIGN_OR_RETURN(CelValue cel_value, - GetGenericProtoAccessApisInstance().GetField( - field, message_wrapper, - ProtoWrapperTypeOptions::kUnsetProtoDefault, - MemoryManagerRef::Pooling(arena))); - Value result; - CEL_RETURN_IF_ERROR(cel::ModernValue(arena, cel_value, result)); - return result; - } - } - return target.GetStruct().GetFieldByName(field, descriptor_pool, - message_factory, arena); -} - -// Helper for StructValue::Qualify. Used for opting out of old reflection -// implementation. -absl::StatusOr> WrappedStructQualify( - const StructValue& struct_value, - absl::Span qualifiers, bool presence_test, - const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, - google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, - bool enable_use_new_field_select_implementation) { - if (!enable_use_new_field_select_implementation) { - if (const google::protobuf::Message* message = - cel::interop_internal::GetLegacyMessage(struct_value); - message != nullptr) { - CelValue::MessageWrapper message_wrapper( - message, &GetGenericProtoTypeInfoInstance()); - CEL_ASSIGN_OR_RETURN(auto legacy_result, - GetGenericProtoAccessApisInstance().Qualify( - qualifiers, message_wrapper, presence_test, - MemoryManagerRef::Pooling(arena))); - Value result; - CEL_RETURN_IF_ERROR(cel::ModernValue(arena, legacy_result.value, result)); - return std::pair{std::move(result), - legacy_result.qualifier_count}; - } - } - return struct_value.Qualify(qualifiers, presence_test, descriptor_pool, - message_factory, arena); -} - absl::StatusOr ApplyQualifier( const Value& operand, const SelectQualifier& qualifier, const google::protobuf::DescriptorPool* absl_nonnull descriptor_pool, google::protobuf::MessageFactory* absl_nonnull message_factory, - google::protobuf::Arena* absl_nonnull arena, - bool enable_use_new_field_select_implementation) { + google::protobuf::Arena* absl_nonnull arena) { return absl::visit( absl::Overload( [&](const FieldSpecifier& field_specifier) -> absl::StatusOr { @@ -347,9 +282,8 @@ absl::StatusOr ApplyQualifier( "